def generate_fix_script(self, report, writer):
"""Generate a script with progress and zfs destroy commands
@note will only generate commands for one host"""
mss = self._config.min_snapshot_size or 0
if mss:
mss = size_to_int(mss)
# end handle type conversion
fss = self._config.min_filesystem_size or 0
if fss:
fss = size_to_int(fss)
# end handle type conversion
sotd = self._config.snapshots_older_than_days
date = datetime_days_ago(time(), sotd)
def predicate(rec):
tn = rec[3]
ctime = rec[4] # creation delta time
size = rec[6]
# We don't know what's part of the Report, and why those values are there.
# This is why we have to re-evaluate the inverse logic to prevent us from
# generating code that would mean something different
return (tn == self.TYPE_SNAPSHOT and (size < mss or ctime <= date)) or (tn == self.TYPE_FILESYSTEM and size < fss)
# end predicate
header_line = "SETTING UP ZFS DESTROY COMMANDS FOR SELECTED DATASETS ON HOST %s\n" % ', '.join(self._config.hosts)
return self._create_zfs_destroy_script(predicate, header_line, report, writer)
def test_size(self):
ki = 1024
for size, res in (('124K', 124.0*ki),
('24.2M', 24.2*ki**2),
('10.6g', 10.6*ki**3), # caps don't matter
('2.75T', 2.75*ki**4),
('5.234P', 5.234*ki**5) ):
assert size_to_int(size) == int(res)
def _sanitize_configuration(self):
"""@return verified and sanitized configuration"""
config = self.configuration()
for size_attr in ('write_chunk_size', 'random_read_volume', 'random_read_chunk_size', 'file_size'):
setattr(config, size_attr, size_to_int(getattr(config, size_attr)))
# end for each attribute
assert config.num_threads >= 1, "Must set at least 1 or more workers"
assert config.output_dir, "Output directory (output_dir) must be set"
assert config.output_dir.isdir(), "output directory at '%s' must be an accessible directory" % self.output_dir
return config
def test_conversion(self):
"""Assure we do it right !"""
size_str = '59.6T'
size = size_to_int(size_str)
assert round(float(size) / 1024**4, 3) == 59.6
now_time = time()
offset = 83921
delta = seconds_to_datetime(now_time) - seconds_to_datetime(now_time - offset)
assert delta_to_seconds(delta) == offset
# just make the call, no check for now
utc_datetime_to_date_time_string(datetime.utcnow())
def generate(self):
# Create an initial query and map filesystems by basename
rep = self.ReportType(copy.deepcopy(self.report_schema))
now = datetime.now()
config = self.settings_value()
if config.mode not in self.valid_modes:
raise ValueError("Can only support the following modes: %s" % ', '.join(self.valid_modes))
# end handle
rep.columns[4][0] = config.mode == self.MODE_RESERVATION and 'reserved' or 'quota'
query = self._session.query(ZDataset).filter(ZDataset.avail != None).\
filter(ZDataset.zfs_priority != None).\
filter(ZDataset.name.like(config.pool_name + '%/%'))
query = host_filter(config.hosts, ZDataset.host, query)
fs_map = dict()
for fs in query:
if fs.property_is_inherited('zfs_priority'):
continue
fs_map.setdefault((fs.host, fs.url().pool()), list()).append(fs)
# end for each filesystem
distribute_space = config.distribute_space
if distribute_space:
distribute_space = size_to_int(distribute_space)
# end convert space
if distribute_space and config.max_cap:
raise ValueError("Please specify either 'max_cap or 'distribute_space', or set one of them to 0")
# end assure we don't see both
if config.debug_priorities and len(fs_map) > 1:
raise AssertionError("If debug_priorities are used, you muse limit the amount of involved hosts to one")
# end make sure debugging makes sense
for (host, pool), fs_list in fs_map.iteritems():
if config.debug_priorities and len(config.debug_priorities) != len(fs_list):
raise AssertionError("Please specify exactly %i priorities, one for each filesystem, got %i" % (len(fs_list), len(config.debug_priorities)))
# end verify priorities
priorities = config.debug_priorities or [fs.zfs_priority for fs in fs_list]
total_parts = sum(priorities)
pool = self._session.instance_by_url(ZFSURL.new(host, pool))
if distribute_space:
total_alloc = distribute_space
else:
total_alloc = pool.size * (config.max_cap / 100.0)
# end handle total_alloc
for fs, prio in zip(fs_list, priorities):
reserve = (total_alloc / float(total_parts)) * prio
rep.records.append([now - fs.updated_at,
fs,
prio,
fs.used,
reserve,
reserve - fs.used,
reserve - fs.avail,
(fs.used / float(reserve)) * 100.0
])
# end for each filesystem
# end for each pool-host pair
if len(rep.records) > 1:
rep.records.append(rep.aggregate_record())
# end aggregate only if it makes sense
return rep
def generate(self):
"""See class description and schema for more information"""
rep = self.ReportType()
rep.columns.extend(self.report_schema)
max_pool_cap,\
fs_cap,\
min_snapshot_size,\
min_filesystem_size,\
snapshots_older_than_days,\
filesystems_older_than_days,\
name_like,\
hosts = ( self._config.max_pool_cap,
self._config.max_filesystem_cap,
self._config.min_snapshot_size,
self._config.min_filesystem_size,
self._config.snapshots_older_than_days,
self._config.filesystems_older_than_days,
self._config.name_like,
self._config.hosts)
assert name_like, "Name matching must be at least '%%', got nothing"
rappend = rep.records.append
now = datetime.now()
nows = time()
percent_comment = '%s.cap > %.0f%%'
# POOL CAPACITIES
###################
if max_pool_cap <= 100.0:
fill_capage = (ZPool.size - ZPool.free) / (ZPool.size * 1.0)
percent_condition = fill_capage >= max_pool_cap / 100.0
comment = percent_comment % (self.TYPE_POOL, max_pool_cap)
for pool in host_filter(hosts, ZPool.host, self._session.query(ZPool).\
filter(percent_condition).\
filter(ZPool.name.like(name_like)).\
order_by(fill_capage.desc())):
rec = [now - pool.updated_at, pool.host, pool.name, self.TYPE_POOL,'NA', 'NA', pool.size, pool.free,
((pool.size - pool.free) / float(pool.size)) * 100, comment]
rappend(rec)
# end for each that matches
# end check enabled
# FILESYSTEM CAPACITIES
########################
if fs_cap <= 100.0:
filesystem = ZDataset.avail != None
fill_capage = 1.0 - (ZDataset.avail / (ZDataset.avail + ZDataset.used) * 1.0)
percent_condition = fill_capage >= fs_cap / 100.0
comment = percent_comment % (self.TYPE_FILESYSTEM, fs_cap)
for fs in host_filter(hosts, ZDataset.host, self._session.query(ZDataset).\
filter(filesystem).\
filter(ZDataset.name.like(name_like)).\
filter(percent_condition).\
order_by(fill_capage.desc())):
rec = [ now - fs.updated_at, fs.host, fs.name, self.TYPE_FILESYSTEM, fs.creation, now - fs.creation, fs.used, fs.avail,
(1.0 - (fs.avail / float((fs.avail + fs.used)))) * 100.0, comment]
rappend(rec)
# end for each record
# end check enabled
# DATASETS TOO SMALL
#####################
# TODO: implement protection against deleting the last snapshot ! In case it's on, we must assure
# the latest snapshot of a filesystem is not deleted. Could be a complex group + join
def make_records(type_condition, condition, type_name, comment):
for ds in host_filter(hosts, ZDataset.host, self._session.query(ZDataset).\
filter(type_condition).\
filter(ZDataset.name.like(name_like)).\
filter(condition).\
order_by(ZDataset.host)):
rec = [now - ds.updated_at, ds.host, ds.name, type_name, ds.creation, now - ds.creation,
ds.used, ds.avail or 0,
ds.avail and ((ds.used / float(ds.used + ds.avail)) * 100.0) or 100.0, comment]
rappend(rec)
# end for each snapshot
# end make records utility
min_size_fmt = '%s.size < %s'
if min_snapshot_size:
condition = ZDataset.used < size_to_int(min_snapshot_size)
make_records(ZDataset.avail == None, condition, self.TYPE_SNAPSHOT,
min_size_fmt % (self.TYPE_SNAPSHOT, min_snapshot_size))
if min_filesystem_size:
condition = ZDataset.used < size_to_int(min_filesystem_size)
make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM,
min_size_fmt % (self.TYPE_FILESYSTEM, min_filesystem_size))
# end handle item sizes
# DATASET AGE
##############
if snapshots_older_than_days:
condition = ZDataset.creation <= datetime_days_ago(nows, snapshots_older_than_days)
make_records(ZDataset.avail == None, condition, self.TYPE_SNAPSHOT,
'%s.creation older %id' % (self.TYPE_SNAPSHOT, snapshots_older_than_days))
if filesystems_older_than_days:
condition = ZDataset.creation <= datetime_days_ago(nows, filesystems_older_than_days)
make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM,
'%s.creation older %id' % (self.TYPE_FILESYSTEM, filesystems_older_than_days))
# end ignore age if time is 0, which is essentially just no
# FILESYSTEM FREE SPACE
########################
min_filesystem_avail_size = size_to_int(self._config.min_filesystem_avail_size)
if min_filesystem_avail_size:
condition = ZDataset.avail >= min_filesystem_avail_size
make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM,
'%s.avail >= %s' % (self.TYPE_FILESYSTEM, self._config.min_filesystem_avail_size))
# end ignore this value if zero
# AGGREGATE
#############
self._aggregate_records(rep.records, now)
return rep
def generate(self):
# Create an initial query and map filesystems by basename
rep = self.ReportType(self.report_schema)
now = datetime.now()
config = self.settings_value()
query = self._session.query(ZDataset).filter(ZDataset.type == self.TYPE_FILESYSTEM).\
filter(ZDataset.name.like(config.name_like)).\
order_by(ZDataset.name, ZDataset.creation)
bmap = dict()
for inst in query:
# Ignore pools and non-leaf filesystems
if inst.is_pool_filesystem() or list(inst.children()):
continue
# end ignore non-leafs
bmap.setdefault(os.path.basename(inst.name), list()).append(inst)
# end build lookup
min_copies, hosts, min_equivalence, ignore_size = config.min_copies,\
config.hosts,\
config.min_equivalence,\
size_to_int(config.ignore_filesystems_smaller)
radd = lambda r: rep.records.append(r) or r
# Sort the list by num-copies-descending
slist = sorted(bmap.items(), key=lambda t: len(t[1]), reverse=True)
# Then compute the equivalence and build the report
for basename, filesystems in slist:
# For now, the oldest one is the master, which owns the copies
# The host filter is applied here
master = filesystems.pop(0)
if min_copies and len(filesystems) + 1 >= min_copies or\
hosts and master.host not in hosts or\
master.used < ignore_size:
continue
# end skip if we are below threshold
mrec = radd([now - master.updated_at,
(not len(filesystems) and '+' or '+-'),
master,
len(filesystems) + 1,
0.0, # done later as avg
None, # never set
master.used,
master.ratio,
master.used * master.ratio])
crecs = list()
msnapshots = list(master.snapshots())
msnapshot_names = [ss.snapshot_name() for ss in msnapshots]
for shadow in filesystems:
equivalence, ss = self._compute_equivalence(msnapshots, msnapshot_names, shadow)
crecs.append(radd([now - shadow.updated_at,
' `-',
shadow,
0,
equivalence * 100.0,
ss,
shadow.used,
shadow.ratio,
shadow.used * shadow.ratio,
]))
# end for each copy
if filesystems:
mrec[4] = sum(r[4] for r in crecs) / len(crecs)
# end compute average equivalence
# drop previous records if equivalence is good enough
if min_equivalence and mrec[4] >= min_equivalence:
rep.records = rep.records[:-len(crecs) - 1]
# end handle
return rep