def calculateSpaceRequired(self, ctx: ThaniyaBackupContext) -> int: # process root directory nErrorsWikiRoot, nSizeTypo3Root = ThaniyaTar.tarCalculateSize( ctx, jk_pathpatternmatcher2.walk( self.__typo3Helper.typo3BaseDirPath, ignoreDirPathPatterns=self.__ignoreDirPathPatterns)) ctx.log.info("I/O expected for the local installation: " + jk_utils.formatBytes(nSizeTypo3Root)) # process database directory dbHost, dbPort, dbName, dbUser, dbPwd = self.__getMySQLDBParameters() assert dbHost in ["127.0.0.1", "localhost"] assert dbPort == 3306 nSizeDB = ThaniyaMySQL.mySQLDumpCalculateSize( ctx=ctx, dbName=dbName, dbUserName=dbUser, dbPassword=dbPwd, ) ctx.log.info("I/O expected for the database: " + jk_utils.formatBytes(nSizeDB)) return nSizeTypo3Root + nSizeDB
def calculateSpaceRequired(self, ctx: ThaniyaBackupContext) -> int: # process root directory nErrorsWikiRoot, nSizeWikiRoot = ThaniyaTar.tarCalculateSize( ctx, jk_pathpatternmatcher2.walk( self.__mwHelper.wikiDirPath, ignoreDirPathPatterns=self.__ignoreDirPathPatterns)) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSizeWikiRoot)) # process database directory nErrorsDBRoot, nSizeDBRoot = ThaniyaTar.tarCalculateSize( ctx, jk_pathpatternmatcher2.walk(self.__mwHelper.wikiDBDirPath)) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSizeDBRoot)) # process cron script nErrorsCronScript, nSizeCronScript = ThaniyaTar.tarCalculateSize( ctx, jk_pathpatternmatcher2.walk(self.__mwHelper.cronScriptFilePath)) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSizeCronScript)) # process start cron script nErrorsStartCronScript, nSizeStartCronScript = ThaniyaTar.tarCalculateSize( ctx, jk_pathpatternmatcher2.walk( self.__mwHelper.startCronScriptFilePath)) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSizeStartCronScript)) return nSizeWikiRoot + nSizeDBRoot + nSizeCronScript + nSizeStartCronScript
def compress(compression: str): outFilePath = OUTPUT_BASE_FILE_NAME + ( ("." + compression) if compression else "") t0 = time.time() with jk_uploadpack.Packer(outFilePath, compression) as up: up.fileGroup("default").bCleanDir = True allFiles = [] for e in jk_pathpatternmatcher2.walk( SRC_DIR_PATH, acceptDirPathPatterns=None, acceptFilePathPatterns="**/*", acceptLinkPathPatterns=None, ignorePathPatterns="**/__*", ignoreDirPathPatterns=None, ignoreFilePathPatterns=None, ignoreLinkPathPatterns=None, emitDirs=False, emitFiles=True, emitLinks=False, emitBaseDirs=False, recursive=True, sort=True, emitErrorEntries=True, clazz=None, ioAdapter=None, ): allFiles.append((e.fullPath, e.relFilePath)) sp = jk_terminal_essentials.Spinner(len(allFiles)) for fullPath, relFilePath in allFiles: sp.spin("packing", relFilePath) up.fileGroup("default").addFile(fullPath, relFilePath) sp.hide() print() print("upload pack", compression if compression else "uncompressed") print() print("\ttotalSizeLogical =", jk_utils.formatBytes(up.totalSizeLogical)) print("\ttotalSizeUncompressed =", jk_utils.formatBytes(up.totalSizeUncompressed)) print("\ttotalSizeCompressed =", jk_utils.formatBytes(up.totalSizeCompressed)) print("\tduration:", jk_utils.formatTime(time.time() - t0, withMilliseconds=True)) print()
def __perform_calcDiskSpaceRequired( self, bd2: BD2, backupTasks: typing.List[AbstractThaniyaTask]) -> int: with ProcessingContext(text="Calculating disk space required", bd2=bd2, bMeasureDuration=True, statsDurationKey="d0_calcDiskSpace") as ctx: nExpectedBytesToWrite = 0 for job in backupTasks: #assert isinstance(job, AbstractThaniyaTask) Assert.isInstance(job, AbstractThaniyaTask) nestedCtx = ctx.descend(job.logMessageCalculateSpaceRequired) with nestedCtx.log as nestedLog: nExpectedBytesToWrite += job.calculateSpaceRequired( nestedCtx) ctx.log.info("Estimated total size of backup: " + jk_utils.formatBytes(nExpectedBytesToWrite)) bd2.statsContainer.setValue("expectedBytesToWrite", nExpectedBytesToWrite) # ---- ctx.log.notice("Done.") return nExpectedBytesToWrite
def calculateSpaceRequired(self, ctx: ThaniyaBackupContext) -> int: nErrors, nSize = ThaniyaTar.tarCalculateSize( ctx=ctx, walker=jk_pathpatternmatcher2.walk(self.__sourceDirPath)) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSize)) return nSize
def processElement(self, ctx: Context, f): ctx.printDetail(self, "=" * 120) if f is None: ctx.printDetail(self, "(none)") else: ctx.printDetail(self, ctx.__class__.__name__ + ": " + f.relFilePath) ctx.printDetail(self, "Size: " + jk_utils.formatBytes(f.getFileSize())) ctx.printDetail(self, "isBinary: " + str(f.isBinary)) ctx.printDetail(self, "isText: " + str(f.isText)) ctx.printDetail(self, "-" * 120) if getattr(f, "isText", False): text = f.readText() for s in text.split("\n"): ctx.printDetail(self, s) elif getattr(f, "isBinary", False): hexRowSize = 16 raw = f.readBinary() sRawHex = raw.hex() n = len(raw) i = 0 bufHex = [] bufASCII = [] while i < n: addr = i.to_bytes(4, byteorder="big").hex() for j in range(0, hexRowSize): iPos1 = i + j iPos2 = iPos1 * 2 if iPos1 < n: bufHex.append(sRawHex[iPos2:iPos2 + 2]) c = raw[iPos1] if 32 <= c <= 126: bufASCII.append(chr(c)) else: bufASCII.append(".") else: bufHex.append(" ") bufASCII.append(" ") s = addr + " " + " ".join(bufHex) + " |" + "".join( bufASCII) + "|" ctx.printDetail(self, s) i += hexRowSize * 2 bufHex.clear() bufASCII.clear() else: ctx.printDetail(self, "(unknown content)") ctx.printDetail(self, "=" * 120) return f
def getFileSize(ctx: ThaniyaBackupContext, filePath: str) -> int: assert isinstance(ctx, ThaniyaBackupContext) assert isinstance(filePath, str) assert filePath assert os.path.isabs(filePath) assert os.path.isfile(filePath) with ctx.descend("Getting size of file: " + repr(filePath)) as ctx: n = os.path.getsize(filePath) ctx.log.notice("Size of " + repr(filePath) + ": " + jk_utils.formatBytes(n)) return n
def getDirTreeSize(ctx: ThaniyaBackupContext, dirPath: str) -> int: assert isinstance(ctx, ThaniyaBackupContext) assert isinstance(dirPath, str) assert dirPath assert os.path.isabs(dirPath) assert os.path.isdir(dirPath) ctx = ctx.descend("Calculating size of directory: " + repr(dirPath)) with ctx.log as nestedLog: n = jk_utils.fsutils.getFolderSize(dirPath) nestedLog.notice("Size of " + repr(dirPath) + ": " + jk_utils.formatBytes(n)) return n
def getSizeOfDevice(ctx: ThaniyaBackupContext, devicePath: str) -> typing.Union[int, None]: assert devicePath.startswith("/dev/") with ctx.log as nestedLog: try: fd = os.open(devicePath, os.O_RDONLY) try: n = os.lseek(fd, 0, os.SEEK_END) nestedLog.notice("Size: " + jk_utils.formatBytes(n)) return n finally: os.close(fd) except: raise Exception("Failed to access device: " + devicePath)
def __perform_backup(self, bd2: BD2, backupTasks: typing.List[AbstractThaniyaTask]): # NOTE: we need to access this context later as it calculates the duration and we need this information separately to log it. processingContext = ProcessingContext(text="Writing the backup data", bd2=bd2, bMeasureDuration=True, statsDurationKey="d2_backup") with processingContext as ctx: for job in backupTasks: Assert.isInstance(job, AbstractThaniyaTask) with ctx.descend(job.logMessagePerformBackup) as nestedCtx: job.performBackup(nestedCtx) ctx.log.notice("All backup tasks completed.") # calculate statistics with ctx.log.descend( "Calculating size of backup performed ...") as nestedLog: nTotalBytesWritten = jk_utils.fsutils.getFolderSize( bd2.effectiveTargetDirPath) fDuration = processingContext.duration if (nTotalBytesWritten > 0) and (fDuration > 0): fAvgWritingSpeed = nTotalBytesWritten / fDuration sAvgWritingSpeed = jk_utils.formatBytesPerSecond( fAvgWritingSpeed) else: fAvgWritingSpeed = None sAvgWritingSpeed = "n/a" ctx.log.info("Total bytes written: " + jk_utils.formatBytes(nTotalBytesWritten)) ctx.log.info("Average writing speed: " + sAvgWritingSpeed) bd2.statsContainer.setValue("totalBytesWritten", nTotalBytesWritten) bd2.statsContainer.setValue("avgWritingSpeed", fAvgWritingSpeed) # ---- ctx.log.notice("Done.")
def __getDirTreeSize(self, dirPath: str, log: jk_logging.AbstractLogger) -> int: assert dirPath assert os.path.isabs(dirPath) assert os.path.isdir(dirPath) nestedLog = log.descend("Calculating size of directory: " + repr(dirPath)) try: n = jk_utils.fsutils.getFolderSize(dirPath) except Exception as ee: nestedLog.exception(ee) raise nestedLog.notice("Size of " + repr(dirPath) + ": " + jk_utils.formatBytes(n)) return n
def _formatBytes(v: float) -> str: if v < 0: return "---" s = jk_utils.formatBytes(v) return s[:-1] + " " + s[-1] + "B"
#!/usr/bin/python3 import time import os import typing import jk_utils import jk_terminal_essentials import jk_uploadpack INPUT_FILE_NAME = "pack.up" with jk_uploadpack.Unpacker(INPUT_FILE_NAME) as up: sp = jk_terminal_essentials.Spinner(len(up.fileGroup("default").files)) up.fileGroup("default").unpackToDir("out", sp) sp.hide() print() print("totalSizePacked =", jk_utils.formatBytes(up.totalSizePacked)) print("totalSizeUnpacked =", jk_utils.formatBytes(up.totalSizeUnpacked)) print()
def calculateSpaceRequired(self, ctx: ThaniyaBackupContext) -> int: nSize = ThaniyaIO.getSizeOfDevice(ctx, self.__devicePath) ctx.log.info("I/O expected: " + jk_utils.formatBytes(nSize)) return nSize
def performBackup(self, backupTasks: list, bSimulate: bool): for x in backupTasks: assert isinstance(x, AbstractThaniyaTask) #Assert.isInstance(x, AbstractThaniyaTask) mainLog = jk_logging.MulticastLogger.create( jk_logging.ConsoleLogger.create( logMsgFormatter=jk_logging.COLOR_LOG_MESSAGE_FORMATTER), jk_logging.BufferLogger.create()) bError = False try: statsContainer = { "tStart": time.time(), "tEnd": None, "success": None, "expectedBytesToWrite": None, "totalBytesWritten": None, "avgWritingSpeed": None, "simulate": bSimulate, } effectiveTargetDirPath = None with ProcessingContext( "Performing backup simulation" if bSimulate else "Performing backup", None, mainLog) as ctxMain: # -------------------------------------------------------------------------------------------------------------------------------- # >>>> estimate the number of bytes we will likely have to write for this backup with ProcessingContext( text="Calculating disk space required", targetDirPath=None, log=ctxMain.log, bMeasureDuration=True, statsContainer=statsContainer, statsDurationKey="d0_calcDiskSpace") as ctx: nExpectedBytesToWrite = 0 for job in backupTasks: assert isinstance(job, AbstractThaniyaTask) #Assert.isInstance(job, AbstractThaniyaTask) nestedCtx = ctx.descend( job.logMessageCalculateSpaceRequired) with nestedCtx.log as nestedLog: nExpectedBytesToWrite += job.calculateSpaceRequired( nestedCtx) ctx.log.info("Estimated total size of backup: " + jk_utils.formatBytes(nExpectedBytesToWrite)) statsContainer[ "expectedBytesToWrite"] = nExpectedBytesToWrite # -------------------------------------------------------------------------------------------------------------------------------- # >>>> now connect to the backup repository with ProcessingContext( text= "Connecting to backup repository and preparing backup", targetDirPath=None, log=ctxMain.log, bMeasureDuration=True, statsContainer=statsContainer, statsDurationKey="d1_connectAndPrepare") as ctx: # check if there is a suitable directory where we can mount the remote file system ThaniyaIO.checkThatDirExists(ctx, self.__mountDirPath) ThaniyaIO.ensureDirMode(ctx, self.__mountDirPath, jk_utils.ChModValue("rwx------")) # mount the remote file system self.__backupConnector.initialize( ctx, self.__mountDirPath, nExpectedBytesToWrite, self.__backupConnectorParameters) if not self.__backupConnector.isReady: raise Exception( "Backup client unexpectedly not ready for writing!" ) # select the target directory where we will store the data. the variable "effectiveTargetDirPath" # will receive the directory selected by the target directory strategy. we will write data there. effectiveTargetDirPath = self.__targetDirStrategy.selectEffectiveTargetDirectory( self.__mountDirPath) ctx.log.info("Selected target directory: " + repr(effectiveTargetDirPath)) # verify that we have the correct directory: the "effectiveTargetDirPath" must be lokated somewhere within # the mounted directory tree. if effectiveTargetDirPath.endswith("/"): effectiveTargetDirPath2 = effectiveTargetDirPath else: effectiveTargetDirPath2 = effectiveTargetDirPath + "/" assert effectiveTargetDirPath2[:len( self.__mountDirPath2)] == self.__mountDirPath2 ctx.log.notice("Creating subdirectories if necessary ...") ThaniyaIO.ensureDirExists(ctx, effectiveTargetDirPath, jk_utils.ChModValue("rwx------")) # check that the target directory fits our requirements: it must be empty. bIsEmpty, contentEntries = ThaniyaIO.checkIfDirIsEmpty( ctx, effectiveTargetDirPath) if not bIsEmpty: print(contentEntries) if STATS_JSON_FILE_NAME in contentEntries: # target directory already seems to contain a backup ctx.log.warn( "Target directory already seems to contain a backup: " + effectiveTargetDirPath2) ctx.log.warn("Overwriting this backup.") else: raise Exception( "Backup directory contains various non-backup files or directories!" ) # now we are ready. but before we begin doing something let's write the backup stats first. jk_json.saveToFilePretty( statsContainer, os.path.join(effectiveTargetDirPath, STATS_JSON_FILE_NAME)) # ---- ctx.log.notice("Done.") # -------------------------------------------------------------------------------------------------------------------------------- # >>>> Writing the backup data if not bSimulate: with ProcessingContext( text="Writing the backup data", targetDirPath=effectiveTargetDirPath, log=ctxMain.log, bMeasureDuration=True, statsContainer=statsContainer, statsDurationKey="d2_backup") as ctx: for job in backupTasks: assert isinstance(job, AbstractThaniyaTask) #Assert.isInstance(job, AbstractThaniyaTask) nestedCtx = ctx.descend( job.logMessagePerformBackup) with nestedCtx.log as nestedLog: job.performBackup(nestedCtx) nTotalBytesWritten = self.__getDirTreeSize( effectiveTargetDirPath, ctx.log) fDuration = ctx.duration if (nTotalBytesWritten > 0) and (fDuration > 0): fAvgWritingSpeed = nTotalBytesWritten / fDuration sAvgWritingSpeed = jk_utils.formatBytesPerSecond( fAvgWritingSpeed) else: fAvgWritingSpeed = None sAvgWritingSpeed = "n/a" ctx.log.info("Total bytes written: " + jk_utils.formatBytes(nTotalBytesWritten)) ctx.log.info("Average writing speed: " + sAvgWritingSpeed) statsContainer[ "totalBytesWritten"] = nTotalBytesWritten statsContainer["avgWritingSpeed"] = fAvgWritingSpeed except ProcessingFallThroughError as ee: bError = True except Exception as ee: bError = True mainLog.error(ee) # -------------------------------------------------------------------------------------------------------------------------------- # >>>> Finish try: # detecting errors detectionLogger = self.__analyseLogMessages(mainLog) if detectionLogger.hasError() or detectionLogger.hasStdErr( ) or detectionLogger.hasException(): bError = True # writing final status log message if bError: mainLog.error("Backup terminated erroneously.") else: mainLog.success("Backup successfully completed.") if effectiveTargetDirPath is not None: # let's try to write the backup stats before termination. statsContainer["tEnd"] = time.time() statsContainer["success"] = not bError jk_json.saveToFilePretty( statsContainer, os.path.join(effectiveTargetDirPath, STATS_JSON_FILE_NAME)) # let's try to write the backup log before termination. bufferLogger = self.__getBufferLogger(mainLog) self.__writeLogToFiles( bufferLogger, os.path.join(effectiveTargetDirPath, PLAINTEXT_LOG_FILE_NAME), os.path.join(effectiveTargetDirPath, JSON_LOG_FILE_NAME)) except ProcessingFallThroughError as ee: bError = True except Exception as ee: bError = True mainLog.error(ee) # terminate connection try: with ProcessingContext("Terminating connection", None, mainLog) as ctxMain: self.__backupConnector.deinitialize(ctx, bError, statsContainer) except ProcessingFallThroughError as ee: bError = True except Exception as ee: bError = True mainLog.error(ee)
#!/usr/bin/python3 import os import time import timeit import jk_utils DIRECTORY_TO_SCAN = os.path.abspath("..") print("Scanning:", DIRECTORY_TO_SCAN) n = jk_utils.fsutils.getFolderSize(DIRECTORY_TO_SCAN) print(n) print(jk_utils.formatBytes(n)) n = jk_utils.fsutils.getFolderSize(DIRECTORY_TO_SCAN, mode="exact") print(n) print(jk_utils.formatBytes(n))