def list_subvolume(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.list(_fuse.getOption("subvol_list_with_stats"))
_fuse.operations.destroy()
return
def compress_subvol_tables(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.compress_all_non_root_tables()
_fuse.operations.destroy()
return
def count_snapshot_created_today(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.count_today_created_subvols(True)
_fuse.operations.destroy()
return
def list_subvolume(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.list(_fuse.getOption("subvol_list_with_stats"))
_fuse.operations.destroy()
return
def calc_subvol_diff(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.get_root_diff(options.subvol_diff.encode('utf8'))
_fuse.operations.destroy()
return
def count_snapshot_created_today(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.count_today_created_subvols(True)
_fuse.operations.destroy()
return
def compress_subvol_tables(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.compress_all_non_root_tables()
_fuse.operations.destroy()
return
def calc_subvol_diff(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setReadonly(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.get_root_diff(options.subvol_diff.encode('utf8'))
_fuse.operations.destroy()
return
def remove_subvolume(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setOption("use_transactions", True)
_fuse.setReadonly(False)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.remove(options.subvol_remove.encode('utf8'))
_fuse.operations.destroy()
return
def remove_subvolume(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
_fuse.setOption("gc_umount_enabled", False)
_fuse.setOption("gc_vacuum_enabled", False)
_fuse.setOption("gc_enabled", False)
_fuse.setOption("use_transactions", True)
_fuse.setReadonly(False)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
sv.remove(options.subvol_remove.encode('utf8'))
_fuse.operations.destroy()
return
def report_disk_usage(self): # {{{3
from dedupsqlfs.fuse.subvolume import Subvolume
subv = Subvolume(self.operations)
manager = self.operations.getManager()
self.getLogger().info("--" * 39)
tableSubvol = manager.getTable("subvolume", True)
disk_usage = 0
disk_usage += manager.getTable("hash", True).getFileSize()
disk_usage += manager.getTable("hash_compression_type", True).getFileSize()
disk_usage += manager.getTable("hash_sizes", True).getFileSize()
disk_usage += manager.getTable("compression_type", True).getFileSize()
disk_usage += manager.getTable("name", True).getFileSize()
disk_usage += manager.getTable("name_pattern_option", True).getFileSize()
disk_usage += manager.getTable("option", True).getFileSize()
disk_usage += manager.getTable("subvolume", True).getFileSize()
disk_usage += manager.getTable("block", True).getFileSize()
apparentSize = 0
dataSize = 0
uniqueSize = 0
compressedSize = 0
compressedUniqueSize = 0
compMethods = {}
for subvol_id in tableSubvol.get_ids():
subvol = tableSubvol.get(subvol_id)
apparentSize += subv.get_apparent_size_fast(subvol["name"])
disk_usage += manager.getTable("inode_%s" % subvol["id"], True).getFileSize()
disk_usage += manager.getTable("inode_option_%s" % subvol["id"], True).getFileSize()
disk_usage += manager.getTable("inode_hash_block_%s" % subvol["id"], True).getFileSize()
disk_usage += manager.getTable("link_%s" % subvol["id"], True).getFileSize()
disk_usage += manager.getTable("xattr_%s" % subvol["id"], True).getFileSize()
disk_usage += manager.getTable("tree_%s" % subvol["id"], True).getFileSize()
tableHCT = manager.getTable('hash_compression_type', True)
tableHS = manager.getTable('hash_sizes', True)
hashCount = subv.prepareIndexHashIdCount()
hashIds = tuple(hashCount.keys())
current = 0
pageSize = 20000
while True:
items = hashIds[current:current+pageSize]
if not len(items):
break
current += pageSize
hash_ids = ",".join(set(str(item) for item in items))
hashTypes = tableHCT.get_types_by_hash_ids(hash_ids)
hashSizes = tableHS.get_sizes_by_hash_ids(hash_ids)
for hash_id in items:
hash_cnt = hashCount[ hash_id ]
method = self.operations.getCompressionTypeName(hashTypes[ hash_id ])
compMethods[ method ] = compMethods.get(method, 0) + 1
hszItem = hashSizes[ hash_id ]
uniqueSize += hszItem[0]
compressedUniqueSize += hszItem[1]
dataSize += hszItem[0]*hash_cnt
compressedSize += hszItem[1]*hash_cnt
sparseSize = apparentSize - dataSize
dedupSize = dataSize - uniqueSize
count_all = 0
comp_types = {}
for method, cnt in compMethods.items():
count_all += cnt
comp_types[ cnt ] = method
if apparentSize:
self.getLogger().info("Apparent size is %s, unique %s.",
format_size(apparentSize), format_size(uniqueSize)
)
self.getLogger().info("Deduped size is %s, ratio is %.2f%%.",
format_size(dedupSize),
100.0 * dedupSize / apparentSize)
self.getLogger().info("Sparse size is %s, ratio is %.2f%%.",
format_size(sparseSize),
100.0 * sparseSize / apparentSize)
self.getLogger().info("Databases take up %s, ratio is %.2f%%.",
format_size(disk_usage),
100.0 * disk_usage / uniqueSize)
self.getLogger().info("Compressed data take up %s:\n- unique %s,\n- saved apparent space is %.2f%%,\n- use of database space: %.2f%%).",
format_size(compressedSize), format_size(compressedUniqueSize),
100.0 * (apparentSize - compressedSize) / apparentSize,
100.0 * compressedUniqueSize / disk_usage,
)
self.getLogger().info("Meta data and indexes take up %s:\n- ratio over apparent is %.2f%%,\n- use of database space: %.2f%%).",
format_size(disk_usage - compressedUniqueSize),
100.0 * (disk_usage - compressedUniqueSize) / uniqueSize,
100.0 * (disk_usage - compressedUniqueSize) / disk_usage,
)
else:
self.getLogger().info("Apparent size is %s.",
format_size(apparentSize)
)
self.getLogger().info("Compressed size is %s.",
format_size(compressedSize)
)
self.getLogger().info("Databases take up %s.",
format_size(disk_usage)
)
if compressedSize:
self.getLogger().info("--" * 39)
self.getLogger().info("Compression by types:")
keys = list(comp_types.keys())
keys.sort(reverse=True)
for key in keys:
compression = comp_types[key]
self.getLogger().info(" %8s used by %.2f%% blocks",
compression, 100.0 * int(key) / count_all
)
return
def do_recompress(options, _fuse):
"""
@param options: Commandline options
@type options: object
@param _fuse: FUSE wrapper
@type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS
"""
tableHash = _fuse.operations.getTable("hash")
tableHashCT = _fuse.operations.getTable("hash_compression_type")
tableBlock = _fuse.operations.getTable("block")
tableSubvol = _fuse.operations.getTable("subvolume")
hashCount = tableHash.get_count()
if _fuse.getOption("verbosity") > 0:
print("Ready to rehash %s blocks." % hashCount)
cur = tableHash.getCursor(True)
cur.execute("SELECT `id` FROM `%s`" % tableHash.getName())
cnt = upd = 0
cpu_n = cpu_count()
lastPrc = ""
_fuse.operations.getManager().setAutocommit(False)
tableBlock.begin()
tableHashCT.begin()
_fuse.operations.getManager().setAutocommit(True)
try:
toCompress = {}
toCompressM = {}
for hashItem in iter(cur.fetchone, None):
cnt += 1
hashId = hashItem["id"]
blockItem = tableBlock.get(hashId)
hashCT = tableHashCT.get(hashId)
curMethod = _fuse.operations.getCompressionTypeName(hashCT["type_id"])
blockData = _fuse.decompressData(curMethod, blockItem["data"])
toCompress[ hashId ] = blockData
toCompressM[ hashId ] = curMethod
if cnt % cpu_n == 0:
for hashId, item in _fuse.compressData(toCompress):
cData, cMethod = item
curMethod = toCompressM[ hashId ]
if cMethod != curMethod:
cMethodId = _fuse.operations.getCompressionTypeId(cMethod)
res = tableBlock.update(hashId, cData)
res2 = tableHashCT.update(hashId, cMethodId)
if res and res2:
upd += 1
toCompress = {}
toCompressM = {}
prc = "%6.2f%%" % (cnt*100.0/hashCount)
if prc != lastPrc:
lastPrc = prc
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\r%s " % prc)
sys.stdout.flush()
if len(toCompress.keys()):
for hashId, item in _fuse.compressData(toCompress):
cData, cMethod = item
curMethod = toCompressM[hashId]
if cMethod != curMethod:
cMethodId = _fuse.operations.getCompressionTypeId(cMethod)
res = tableBlock.update(hashId, cData)
res2 = tableHashCT.update(hashId, cMethodId)
if res and res2:
upd += 1
except:
pass
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\n")
sys.stdout.flush()
if _fuse.getOption("verbosity") > 0:
print("Processed %s blocks, recompressed %s blocks." % (cnt, upd,))
if hashCount != cnt:
_fuse.operations.getManager().setAutocommit(False)
tableBlock.rollback()
tableHashCT.rollback()
_fuse.operations.getManager().setAutocommit(True)
print("Something went wrong? Changes are rolled back!")
return 1
_fuse.operations.getManager().setAutocommit(False)
tableBlock.commit()
tableHashCT.commit()
_fuse.operations.getManager().setAutocommit(True)
subvCount = tableSubvol.get_count()
if _fuse.getOption("verbosity") > 0:
print("Recalculate filesystem and %s subvolumes statistics." % subvCount)
cur = tableSubvol.getCursor(True)
cur.execute("SELECT * FROM `%s`" % tableSubvol.getName())
_fuse.operations.getManager().setAutocommit(False)
tableSubvol.begin()
_fuse.operations.getManager().setAutocommit(True)
from dedupsqlfs.fuse.subvolume import Subvolume
sv = Subvolume(_fuse.operations)
cnt = 0
lastPrc = ""
for subvItem in iter(cur.fetchone, None):
sv.clean_stats(subvItem["name"])
cnt += 1
prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3)
if prc != lastPrc:
lastPrc = prc
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\r%s " % prc)
sys.stdout.flush()
sv.get_usage(subvItem["name"], True)
cnt += 1
prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3)
if prc != lastPrc:
lastPrc = prc
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\r%s " % prc)
sys.stdout.flush()
sv.get_root_diff(subvItem["name"])
cnt += 1
prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3)
if prc != lastPrc:
lastPrc = prc
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\r%s " % prc)
sys.stdout.flush()
if _fuse.getOption("verbosity") > 0:
sys.stdout.write("\n")
sys.stdout.flush()
_fuse.operations.getManager().setAutocommit(False)
tableSubvol.commit()
_fuse.operations.getManager().setAutocommit(True)
return 0
def report_disk_usage(self): # {{{3
from dedupsqlfs.fuse.subvolume import Subvolume
subv = Subvolume(self.operations)
manager = self.operations.getManager()
self.getLogger().info("--" * 39)
tableSubvol = manager.getTable("subvolume", True)
disk_usage = 0
disk_usage += manager.getTable("hash", True).getFileSize()
disk_usage += manager.getTable("hash_compression_type",
True).getFileSize()
disk_usage += manager.getTable("hash_sizes", True).getFileSize()
disk_usage += manager.getTable("compression_type", True).getFileSize()
disk_usage += manager.getTable("name", True).getFileSize()
disk_usage += manager.getTable("name_pattern_option",
True).getFileSize()
disk_usage += manager.getTable("option", True).getFileSize()
disk_usage += manager.getTable("subvolume", True).getFileSize()
disk_usage += manager.getTable("block", True).getFileSize()
apparentSize = 0
dataSize = 0
uniqueSize = 0
compressedSize = 0
compressedUniqueSize = 0
compMethods = {}
for subvol_id in tableSubvol.get_ids():
subvol = tableSubvol.get(subvol_id)
apparentSize += subv.get_apparent_size_fast(subvol["name"])
disk_usage += manager.getTable("inode_" + subvol["hash"],
True).getFileSize()
disk_usage += manager.getTable("inode_option_" + subvol["hash"],
True).getFileSize()
disk_usage += manager.getTable(
"inode_hash_block_" + subvol["hash"], True).getFileSize()
disk_usage += manager.getTable("link_" + subvol["hash"],
True).getFileSize()
disk_usage += manager.getTable("xattr_" + subvol["hash"],
True).getFileSize()
disk_usage += manager.getTable("tree_" + subvol["hash"],
True).getFileSize()
tableHCT = manager.getTable('hash_compression_type', True)
tableHS = manager.getTable('hash_sizes', True)
hashCount = subv.prepareIndexHashIdCount()
hashIds = tuple(hashCount.keys())
current = 0
pageSize = 20000
while True:
items = hashIds[current:current + pageSize]
if not len(items):
break
current += pageSize
hash_ids = ",".join(set(str(item) for item in items))
hashTypes = tableHCT.get_types_by_hash_ids(hash_ids)
hashSizes = tableHS.get_sizes_by_hash_ids(hash_ids)
for hash_id in items:
hash_cnt = hashCount[hash_id]
method = self.operations.getCompressionTypeName(
hashTypes[hash_id])
compMethods[method] = compMethods.get(method, 0) + 1
hszItem = hashSizes[hash_id]
uniqueSize += hszItem[0]
compressedUniqueSize += hszItem[1]
dataSize += hszItem[0] * hash_cnt
compressedSize += hszItem[1] * hash_cnt
sparseSize = apparentSize - dataSize
dedupSize = dataSize - uniqueSize
count_all = 0
comp_types = {}
for method, cnt in compMethods.items():
count_all += cnt
comp_types[cnt] = method
if apparentSize:
self.getLogger().info("Apparent size is %s, unique %s.",
format_size(apparentSize),
format_size(uniqueSize))
self.getLogger().info("Deduped size is %s, ratio is %.2f%%.",
format_size(dedupSize),
100.0 * dedupSize / apparentSize)
self.getLogger().info("Sparse size is %s, ratio is %.2f%%.",
format_size(sparseSize),
100.0 * sparseSize / apparentSize)
self.getLogger().info("Databases take up %s, ratio is %.2f%%.",
format_size(disk_usage),
100.0 * disk_usage / uniqueSize)
self.getLogger().info(
"Compressed data take up %s:\n- unique %s,\n- saved apparent space is %.2f%%,\n- use of database space: %.2f%%).",
format_size(compressedSize),
format_size(compressedUniqueSize),
100.0 * (apparentSize - compressedSize) / apparentSize,
100.0 * compressedUniqueSize / disk_usage,
)
self.getLogger().info(
"Meta data and indexes take up %s:\n- ratio over apparent is %.2f%%,\n- use of database space: %.2f%%).",
format_size(disk_usage - compressedUniqueSize),
100.0 * (disk_usage - compressedUniqueSize) / uniqueSize,
100.0 * (disk_usage - compressedUniqueSize) / disk_usage,
)
else:
self.getLogger().info("Apparent size is %s.",
format_size(apparentSize))
self.getLogger().info("Compressed size is %s.",
format_size(compressedSize))
self.getLogger().info("Databases take up %s.",
format_size(disk_usage))
if compressedSize:
self.getLogger().info("--" * 39)
self.getLogger().info("Compression by types:")
keys = list(comp_types.keys())
keys.sort(reverse=True)
for key in keys:
compression = comp_types[key]
self.getLogger().info(" %8s used by %.2f%% blocks",
compression,
100.0 * int(key) / count_all)
return