def list_subvolume(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ _fuse.setOption("gc_umount_enabled", False) _fuse.setOption("gc_vacuum_enabled", False) _fuse.setOption("gc_enabled", False) _fuse.setReadonly(True) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) sv.list(_fuse.getOption("subvol_list_with_stats")) _fuse.operations.destroy() return
def compress_subvol_tables(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ _fuse.setOption("gc_umount_enabled", False) _fuse.setOption("gc_vacuum_enabled", False) _fuse.setOption("gc_enabled", False) _fuse.setReadonly(True) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) sv.compress_all_non_root_tables() _fuse.operations.destroy() return
def count_snapshot_created_today(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ _fuse.setOption("gc_umount_enabled", False) _fuse.setOption("gc_vacuum_enabled", False) _fuse.setOption("gc_enabled", False) _fuse.setReadonly(True) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) sv.count_today_created_subvols(True) _fuse.operations.destroy() return
def calc_subvol_diff(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ _fuse.setOption("gc_umount_enabled", False) _fuse.setOption("gc_vacuum_enabled", False) _fuse.setOption("gc_enabled", False) _fuse.setReadonly(True) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) sv.get_root_diff(options.subvol_diff.encode('utf8')) _fuse.operations.destroy() return
def remove_subvolume(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ _fuse.setOption("gc_umount_enabled", False) _fuse.setOption("gc_vacuum_enabled", False) _fuse.setOption("gc_enabled", False) _fuse.setOption("use_transactions", True) _fuse.setReadonly(False) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) sv.remove(options.subvol_remove.encode('utf8')) _fuse.operations.destroy() return
def report_disk_usage(self): # {{{3 from dedupsqlfs.fuse.subvolume import Subvolume subv = Subvolume(self.operations) manager = self.operations.getManager() self.getLogger().info("--" * 39) tableSubvol = manager.getTable("subvolume", True) disk_usage = 0 disk_usage += manager.getTable("hash", True).getFileSize() disk_usage += manager.getTable("hash_compression_type", True).getFileSize() disk_usage += manager.getTable("hash_sizes", True).getFileSize() disk_usage += manager.getTable("compression_type", True).getFileSize() disk_usage += manager.getTable("name", True).getFileSize() disk_usage += manager.getTable("name_pattern_option", True).getFileSize() disk_usage += manager.getTable("option", True).getFileSize() disk_usage += manager.getTable("subvolume", True).getFileSize() disk_usage += manager.getTable("block", True).getFileSize() apparentSize = 0 dataSize = 0 uniqueSize = 0 compressedSize = 0 compressedUniqueSize = 0 compMethods = {} for subvol_id in tableSubvol.get_ids(): subvol = tableSubvol.get(subvol_id) apparentSize += subv.get_apparent_size_fast(subvol["name"]) disk_usage += manager.getTable("inode_%s" % subvol["id"], True).getFileSize() disk_usage += manager.getTable("inode_option_%s" % subvol["id"], True).getFileSize() disk_usage += manager.getTable("inode_hash_block_%s" % subvol["id"], True).getFileSize() disk_usage += manager.getTable("link_%s" % subvol["id"], True).getFileSize() disk_usage += manager.getTable("xattr_%s" % subvol["id"], True).getFileSize() disk_usage += manager.getTable("tree_%s" % subvol["id"], True).getFileSize() tableHCT = manager.getTable('hash_compression_type', True) tableHS = manager.getTable('hash_sizes', True) hashCount = subv.prepareIndexHashIdCount() hashIds = tuple(hashCount.keys()) current = 0 pageSize = 20000 while True: items = hashIds[current:current+pageSize] if not len(items): break current += pageSize hash_ids = ",".join(set(str(item) for item in items)) hashTypes = tableHCT.get_types_by_hash_ids(hash_ids) hashSizes = tableHS.get_sizes_by_hash_ids(hash_ids) for hash_id in items: hash_cnt = hashCount[ hash_id ] method = self.operations.getCompressionTypeName(hashTypes[ hash_id ]) compMethods[ method ] = compMethods.get(method, 0) + 1 hszItem = hashSizes[ hash_id ] uniqueSize += hszItem[0] compressedUniqueSize += hszItem[1] dataSize += hszItem[0]*hash_cnt compressedSize += hszItem[1]*hash_cnt sparseSize = apparentSize - dataSize dedupSize = dataSize - uniqueSize count_all = 0 comp_types = {} for method, cnt in compMethods.items(): count_all += cnt comp_types[ cnt ] = method if apparentSize: self.getLogger().info("Apparent size is %s, unique %s.", format_size(apparentSize), format_size(uniqueSize) ) self.getLogger().info("Deduped size is %s, ratio is %.2f%%.", format_size(dedupSize), 100.0 * dedupSize / apparentSize) self.getLogger().info("Sparse size is %s, ratio is %.2f%%.", format_size(sparseSize), 100.0 * sparseSize / apparentSize) self.getLogger().info("Databases take up %s, ratio is %.2f%%.", format_size(disk_usage), 100.0 * disk_usage / uniqueSize) self.getLogger().info("Compressed data take up %s:\n- unique %s,\n- saved apparent space is %.2f%%,\n- use of database space: %.2f%%).", format_size(compressedSize), format_size(compressedUniqueSize), 100.0 * (apparentSize - compressedSize) / apparentSize, 100.0 * compressedUniqueSize / disk_usage, ) self.getLogger().info("Meta data and indexes take up %s:\n- ratio over apparent is %.2f%%,\n- use of database space: %.2f%%).", format_size(disk_usage - compressedUniqueSize), 100.0 * (disk_usage - compressedUniqueSize) / uniqueSize, 100.0 * (disk_usage - compressedUniqueSize) / disk_usage, ) else: self.getLogger().info("Apparent size is %s.", format_size(apparentSize) ) self.getLogger().info("Compressed size is %s.", format_size(compressedSize) ) self.getLogger().info("Databases take up %s.", format_size(disk_usage) ) if compressedSize: self.getLogger().info("--" * 39) self.getLogger().info("Compression by types:") keys = list(comp_types.keys()) keys.sort(reverse=True) for key in keys: compression = comp_types[key] self.getLogger().info(" %8s used by %.2f%% blocks", compression, 100.0 * int(key) / count_all ) return
def do_recompress(options, _fuse): """ @param options: Commandline options @type options: object @param _fuse: FUSE wrapper @type _fuse: dedupsqlfs.fuse.dedupfs.DedupFS """ tableHash = _fuse.operations.getTable("hash") tableHashCT = _fuse.operations.getTable("hash_compression_type") tableBlock = _fuse.operations.getTable("block") tableSubvol = _fuse.operations.getTable("subvolume") hashCount = tableHash.get_count() if _fuse.getOption("verbosity") > 0: print("Ready to rehash %s blocks." % hashCount) cur = tableHash.getCursor(True) cur.execute("SELECT `id` FROM `%s`" % tableHash.getName()) cnt = upd = 0 cpu_n = cpu_count() lastPrc = "" _fuse.operations.getManager().setAutocommit(False) tableBlock.begin() tableHashCT.begin() _fuse.operations.getManager().setAutocommit(True) try: toCompress = {} toCompressM = {} for hashItem in iter(cur.fetchone, None): cnt += 1 hashId = hashItem["id"] blockItem = tableBlock.get(hashId) hashCT = tableHashCT.get(hashId) curMethod = _fuse.operations.getCompressionTypeName(hashCT["type_id"]) blockData = _fuse.decompressData(curMethod, blockItem["data"]) toCompress[ hashId ] = blockData toCompressM[ hashId ] = curMethod if cnt % cpu_n == 0: for hashId, item in _fuse.compressData(toCompress): cData, cMethod = item curMethod = toCompressM[ hashId ] if cMethod != curMethod: cMethodId = _fuse.operations.getCompressionTypeId(cMethod) res = tableBlock.update(hashId, cData) res2 = tableHashCT.update(hashId, cMethodId) if res and res2: upd += 1 toCompress = {} toCompressM = {} prc = "%6.2f%%" % (cnt*100.0/hashCount) if prc != lastPrc: lastPrc = prc if _fuse.getOption("verbosity") > 0: sys.stdout.write("\r%s " % prc) sys.stdout.flush() if len(toCompress.keys()): for hashId, item in _fuse.compressData(toCompress): cData, cMethod = item curMethod = toCompressM[hashId] if cMethod != curMethod: cMethodId = _fuse.operations.getCompressionTypeId(cMethod) res = tableBlock.update(hashId, cData) res2 = tableHashCT.update(hashId, cMethodId) if res and res2: upd += 1 except: pass if _fuse.getOption("verbosity") > 0: sys.stdout.write("\n") sys.stdout.flush() if _fuse.getOption("verbosity") > 0: print("Processed %s blocks, recompressed %s blocks." % (cnt, upd,)) if hashCount != cnt: _fuse.operations.getManager().setAutocommit(False) tableBlock.rollback() tableHashCT.rollback() _fuse.operations.getManager().setAutocommit(True) print("Something went wrong? Changes are rolled back!") return 1 _fuse.operations.getManager().setAutocommit(False) tableBlock.commit() tableHashCT.commit() _fuse.operations.getManager().setAutocommit(True) subvCount = tableSubvol.get_count() if _fuse.getOption("verbosity") > 0: print("Recalculate filesystem and %s subvolumes statistics." % subvCount) cur = tableSubvol.getCursor(True) cur.execute("SELECT * FROM `%s`" % tableSubvol.getName()) _fuse.operations.getManager().setAutocommit(False) tableSubvol.begin() _fuse.operations.getManager().setAutocommit(True) from dedupsqlfs.fuse.subvolume import Subvolume sv = Subvolume(_fuse.operations) cnt = 0 lastPrc = "" for subvItem in iter(cur.fetchone, None): sv.clean_stats(subvItem["name"]) cnt += 1 prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3) if prc != lastPrc: lastPrc = prc if _fuse.getOption("verbosity") > 0: sys.stdout.write("\r%s " % prc) sys.stdout.flush() sv.get_usage(subvItem["name"], True) cnt += 1 prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3) if prc != lastPrc: lastPrc = prc if _fuse.getOption("verbosity") > 0: sys.stdout.write("\r%s " % prc) sys.stdout.flush() sv.get_root_diff(subvItem["name"]) cnt += 1 prc = "%6.2f%%" % (cnt * 100.0 / subvCount / 3) if prc != lastPrc: lastPrc = prc if _fuse.getOption("verbosity") > 0: sys.stdout.write("\r%s " % prc) sys.stdout.flush() if _fuse.getOption("verbosity") > 0: sys.stdout.write("\n") sys.stdout.flush() _fuse.operations.getManager().setAutocommit(False) tableSubvol.commit() _fuse.operations.getManager().setAutocommit(True) return 0
def report_disk_usage(self): # {{{3 from dedupsqlfs.fuse.subvolume import Subvolume subv = Subvolume(self.operations) manager = self.operations.getManager() self.getLogger().info("--" * 39) tableSubvol = manager.getTable("subvolume", True) disk_usage = 0 disk_usage += manager.getTable("hash", True).getFileSize() disk_usage += manager.getTable("hash_compression_type", True).getFileSize() disk_usage += manager.getTable("hash_sizes", True).getFileSize() disk_usage += manager.getTable("compression_type", True).getFileSize() disk_usage += manager.getTable("name", True).getFileSize() disk_usage += manager.getTable("name_pattern_option", True).getFileSize() disk_usage += manager.getTable("option", True).getFileSize() disk_usage += manager.getTable("subvolume", True).getFileSize() disk_usage += manager.getTable("block", True).getFileSize() apparentSize = 0 dataSize = 0 uniqueSize = 0 compressedSize = 0 compressedUniqueSize = 0 compMethods = {} for subvol_id in tableSubvol.get_ids(): subvol = tableSubvol.get(subvol_id) apparentSize += subv.get_apparent_size_fast(subvol["name"]) disk_usage += manager.getTable("inode_" + subvol["hash"], True).getFileSize() disk_usage += manager.getTable("inode_option_" + subvol["hash"], True).getFileSize() disk_usage += manager.getTable( "inode_hash_block_" + subvol["hash"], True).getFileSize() disk_usage += manager.getTable("link_" + subvol["hash"], True).getFileSize() disk_usage += manager.getTable("xattr_" + subvol["hash"], True).getFileSize() disk_usage += manager.getTable("tree_" + subvol["hash"], True).getFileSize() tableHCT = manager.getTable('hash_compression_type', True) tableHS = manager.getTable('hash_sizes', True) hashCount = subv.prepareIndexHashIdCount() hashIds = tuple(hashCount.keys()) current = 0 pageSize = 20000 while True: items = hashIds[current:current + pageSize] if not len(items): break current += pageSize hash_ids = ",".join(set(str(item) for item in items)) hashTypes = tableHCT.get_types_by_hash_ids(hash_ids) hashSizes = tableHS.get_sizes_by_hash_ids(hash_ids) for hash_id in items: hash_cnt = hashCount[hash_id] method = self.operations.getCompressionTypeName( hashTypes[hash_id]) compMethods[method] = compMethods.get(method, 0) + 1 hszItem = hashSizes[hash_id] uniqueSize += hszItem[0] compressedUniqueSize += hszItem[1] dataSize += hszItem[0] * hash_cnt compressedSize += hszItem[1] * hash_cnt sparseSize = apparentSize - dataSize dedupSize = dataSize - uniqueSize count_all = 0 comp_types = {} for method, cnt in compMethods.items(): count_all += cnt comp_types[cnt] = method if apparentSize: self.getLogger().info("Apparent size is %s, unique %s.", format_size(apparentSize), format_size(uniqueSize)) self.getLogger().info("Deduped size is %s, ratio is %.2f%%.", format_size(dedupSize), 100.0 * dedupSize / apparentSize) self.getLogger().info("Sparse size is %s, ratio is %.2f%%.", format_size(sparseSize), 100.0 * sparseSize / apparentSize) self.getLogger().info("Databases take up %s, ratio is %.2f%%.", format_size(disk_usage), 100.0 * disk_usage / uniqueSize) self.getLogger().info( "Compressed data take up %s:\n- unique %s,\n- saved apparent space is %.2f%%,\n- use of database space: %.2f%%).", format_size(compressedSize), format_size(compressedUniqueSize), 100.0 * (apparentSize - compressedSize) / apparentSize, 100.0 * compressedUniqueSize / disk_usage, ) self.getLogger().info( "Meta data and indexes take up %s:\n- ratio over apparent is %.2f%%,\n- use of database space: %.2f%%).", format_size(disk_usage - compressedUniqueSize), 100.0 * (disk_usage - compressedUniqueSize) / uniqueSize, 100.0 * (disk_usage - compressedUniqueSize) / disk_usage, ) else: self.getLogger().info("Apparent size is %s.", format_size(apparentSize)) self.getLogger().info("Compressed size is %s.", format_size(compressedSize)) self.getLogger().info("Databases take up %s.", format_size(disk_usage)) if compressedSize: self.getLogger().info("--" * 39) self.getLogger().info("Compression by types:") keys = list(comp_types.keys()) keys.sort(reverse=True) for key in keys: compression = comp_types[key] self.getLogger().info(" %8s used by %.2f%% blocks", compression, 100.0 * int(key) / count_all) return