def runRestore(self, backupFolder, outputFolder) : if self.lock.acquireLock(): Logger.log("Lock acquired") #try: state = self.loadState() self.restoreFolders(state, backupFolder, outputFolder) #except Exception, e: # Logger.log("Error: %s" % e) # raise e self.lock.releaseLock() else: Logger.log('Could not acquire lock')
def loadState(self): if os.path.exists(self.treeFilePath) : Logger.log("Reading state from %s" % self.treeFilePath) return self.serializer.unserialize(self.treeFilePath) else : Logger.log("Creating new state object") default = State() default.lastCheck = 0 default.lastFileId = 0 default.trees = {} default.treeFilePathEncoded = self.nameManager.encodeName(self.treeFilePath, default) return default
def restore(self, srcEncryptedPath, dstPath) : try: fd=os.open(self.passphrasePath, os.O_RDONLY) cmd='gpg --no-tty --passphrase-fd {fd}'.format(fd=fd) self._createDirsForFile(dstPath) with open(os.path.join(self.backupFolder, srcEncryptedPath), 'r') as stdin_fh: with open(dstPath, 'w') as stdout_fh: proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh) proc.communicate() if proc.returncode != 0: raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode)) if (not os.path.exists(dstPath)) or (os.path.getsize(dstPath) == 0): raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcEncryptedPath, dst=dstPath)) except: raise finally: os.close(fd) Logger.log('Decrypted {src} and saved as {dst}'.format(src=srcEncryptedPath, dst=dstPath))
def backup(self, srcName, dstName) : try: # TODO: remove os-specific / fd=os.open(self.passphrasePath, os.O_RDONLY) dst = os.path.join(self.backupFolder, dstName) cmd='gpg --no-tty --force-mdc --passphrase-fd {fd} -c'.format(fd=fd) self._createDirsForFile(dst) with open(srcName, 'r') as stdin_fh: with open(dst, 'w') as stdout_fh: proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh) proc.communicate() if proc.returncode != 0: raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode)) if (not os.path.exists(dst)) or (os.path.getsize(dst) == 0): raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcName, dst=dst)) except: raise finally: os.close(fd) assert os.path.exists(dst) and os.path.getsize(dst) > 0, 'File {0} not saved correctly'.format(dst) Logger.log('+ Encrypted {src} and saved as {dst}'.format(src=srcName, dst=dst))
def backupFolders(self, state, inputFolders, outputFolder, excludePatterns, stats): totalNumberOfFiles = 0 totalSize = 0 for folder in inputFolders: Logger.log("Listing files in %s" % folder) tree = TreeNodeDirectory.createTreeFromFilesystem(folder, os.path.basename(folder), excludePatterns) if folder in state.trees: state.trees[folder].ensureDstPathOnAll() Logger.log("Comparing files in %s" % folder) (listOfRemoved, listOfAdded, listOfModified) = tree.merge(state.trees[folder]) else: Logger.log("Adding all files from new folder %s" % folder) listOfAdded = tree.getAllFiles() listOfRemoved = [] listOfModified = [] upFolder = os.path.dirname(folder) for f in listOfAdded: self.addFile(upFolder, f, state, stats) if f.dstPath is None: raise Exception('error') for f in listOfModified: self.updateFile(upFolder, f, stats) if f.dstPath is None: raise Exception('error') for f in listOfRemoved: self.removeFile(upFolder, f, stats) if f.dstPath is None: raise Exception('error') tree.ensureDstPathOnAll() state.trees[folder] = tree totalNumberOfFiles += tree.numberOfFiles totalSize += tree.size state.lastCheck = int(time.time()) Logger.log("folder %s:" % folder) Logger.log("Number of files: %d, size: %.2f MB" % (totalNumberOfFiles, totalSize/1024/1024))
def runBackup(self, inputFolders, outputFolder, excludePatterns, updateEvery) : if self.lock.acquireLock(): Logger.log("Lock acquired") #try: state = self.loadState() if self.isUpdatePending(state, updateEvery): Logger.log("update pending") #Logger.log('last successful backup at {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch']).strftime('%Y-%m-%d %H:%M:%S'))) stats = Stats() self.backupFolders(state, inputFolders, outputFolder, excludePatterns, stats) self.printStats(stats) self.saveState(state) else: Logger.log('backup not necessary at this moment') #Logger.log('next no sooner than {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch'] + updateEvery).strftime('%Y-%m-%d %H:%M:%S'))) #except Exception, e: # Logger.log("Error: %s" % e) # raise e self.lock.releaseLock() else: Logger.log('Could not acquire lock')
try: nameManager = HexNumbersNameManager() backupProvider = EncryptedBackupProvider(config.storeBackupFolder, config.passphrasePath) serializer = PickleSerializer() if len(sys.argv) < 2: print 'expected at least one command line argument (e.g. backup, restore or ls)' sys.exit() if sys.argv[1] == 'backup' : controller = TreeStoreBackupController(config.dataPath, lock, serializer, nameManager, backupProvider) controller.runBackup(config.foldersToBackup, config.storeBackupFolder, config.excludePatterns, config.updateBackupEvery) elif sys.argv[1] == 'ls' : state = serializer.unserialize(os.path.join(config.dataPath, 'tree.dat')) for tree in state.trees.values(): for f in tree.getAllFiles(): print f.path elif sys.argv[1] == 'restore' : if len(sys.argv) == 5: pass else: print 'you need to specify input and output folder and passphrase file path for restore' else: print 'unknown command line option (expected backup or ls)' except KeyboardInterrupt: Logger.log('Action aborted by a user') lock.releaseLock()
def updateFile(self, folder, f, stats): assert f.dstPath is not None, "missing dstPath for updated file: " + str(f) Logger.log("Updating or adding file: %s" % f.path) stats.updatedFilesSize += f.size stats.updatedFilesCount += 1 self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def addFile(self, folder, f, state, stats): Logger.log("Updating or adding file: %s" % f.path) stats.addedFilesSize += f.size stats.addedFilesCount += 1 f.dstPath = self.nameManager.encodeName(f.path, state) self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def saveState(self, state): Logger.log("Serializing state to %s" % self.treeFilePath) self.serializer.serialize(self.treeFilePath, state) self.backupProvider.backup(self.treeFilePath, state.treeFilePathEncoded)
def printStats(self, stats): Logger.log("Added %d files (%.2f MB)" % (stats.addedFilesCount, stats.addedFilesSize/1024/1024)) Logger.log("Updated %d files (%.2f MB)" % (stats.updatedFilesCount, stats.updatedFilesSize/1024/1024)) Logger.log("Removed %d files (%.2f MB)" % (stats.removedFilesCount, stats.removedFilesSize/1024/1024))
def removeFile(self, folder, f, stats): assert f.dstPath is not None, "missing dstPath for removed file: " + str(f) Logger.log("Removing file: %s" % f.path) stats.removedFilesSize += f.size stats.removedFilesCount += 1 self.backupProvider.remove(f.dstPath)