def runRestore(self, backupFolder, outputFolder) :
if self.lock.acquireLock():
Logger.log("Lock acquired")
#try:
state = self.loadState()
self.restoreFolders(state, backupFolder, outputFolder)
#except Exception, e:
# Logger.log("Error: %s" % e)
# raise e
self.lock.releaseLock()
else:
Logger.log('Could not acquire lock')
def loadState(self):
if os.path.exists(self.treeFilePath) :
Logger.log("Reading state from %s" % self.treeFilePath)
return self.serializer.unserialize(self.treeFilePath)
else :
Logger.log("Creating new state object")
default = State()
default.lastCheck = 0
default.lastFileId = 0
default.trees = {}
default.treeFilePathEncoded = self.nameManager.encodeName(self.treeFilePath, default)
return default
def restore(self, srcEncryptedPath, dstPath) :
try:
fd=os.open(self.passphrasePath, os.O_RDONLY)
cmd='gpg --no-tty --passphrase-fd {fd}'.format(fd=fd)
self._createDirsForFile(dstPath)
with open(os.path.join(self.backupFolder, srcEncryptedPath), 'r') as stdin_fh:
with open(dstPath, 'w') as stdout_fh:
proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh)
proc.communicate()
if proc.returncode != 0:
raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode))
if (not os.path.exists(dstPath)) or (os.path.getsize(dstPath) == 0):
raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcEncryptedPath, dst=dstPath))
except:
raise
finally:
os.close(fd)
Logger.log('Decrypted {src} and saved as {dst}'.format(src=srcEncryptedPath, dst=dstPath))
def backup(self, srcName, dstName) :
try:
# TODO: remove os-specific /
fd=os.open(self.passphrasePath, os.O_RDONLY)
dst = os.path.join(self.backupFolder, dstName)
cmd='gpg --no-tty --force-mdc --passphrase-fd {fd} -c'.format(fd=fd)
self._createDirsForFile(dst)
with open(srcName, 'r') as stdin_fh:
with open(dst, 'w') as stdout_fh:
proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh)
proc.communicate()
if proc.returncode != 0:
raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode))
if (not os.path.exists(dst)) or (os.path.getsize(dst) == 0):
raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcName, dst=dst))
except:
raise
finally:
os.close(fd)
assert os.path.exists(dst) and os.path.getsize(dst) > 0, 'File {0} not saved correctly'.format(dst)
Logger.log('+ Encrypted {src} and saved as {dst}'.format(src=srcName, dst=dst))
def backupFolders(self, state, inputFolders, outputFolder, excludePatterns, stats):
totalNumberOfFiles = 0
totalSize = 0
for folder in inputFolders:
Logger.log("Listing files in %s" % folder)
tree = TreeNodeDirectory.createTreeFromFilesystem(folder, os.path.basename(folder), excludePatterns)
if folder in state.trees:
state.trees[folder].ensureDstPathOnAll()
Logger.log("Comparing files in %s" % folder)
(listOfRemoved, listOfAdded, listOfModified) = tree.merge(state.trees[folder])
else:
Logger.log("Adding all files from new folder %s" % folder)
listOfAdded = tree.getAllFiles()
listOfRemoved = []
listOfModified = []
upFolder = os.path.dirname(folder)
for f in listOfAdded:
self.addFile(upFolder, f, state, stats)
if f.dstPath is None: raise Exception('error')
for f in listOfModified:
self.updateFile(upFolder, f, stats)
if f.dstPath is None: raise Exception('error')
for f in listOfRemoved:
self.removeFile(upFolder, f, stats)
if f.dstPath is None: raise Exception('error')
tree.ensureDstPathOnAll()
state.trees[folder] = tree
totalNumberOfFiles += tree.numberOfFiles
totalSize += tree.size
state.lastCheck = int(time.time())
Logger.log("folder %s:" % folder)
Logger.log("Number of files: %d, size: %.2f MB" % (totalNumberOfFiles, totalSize/1024/1024))
def runBackup(self, inputFolders, outputFolder, excludePatterns, updateEvery) :
if self.lock.acquireLock():
Logger.log("Lock acquired")
#try:
state = self.loadState()
if self.isUpdatePending(state, updateEvery):
Logger.log("update pending")
#Logger.log('last successful backup at {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch']).strftime('%Y-%m-%d %H:%M:%S')))
stats = Stats()
self.backupFolders(state, inputFolders, outputFolder, excludePatterns, stats)
self.printStats(stats)
self.saveState(state)
else:
Logger.log('backup not necessary at this moment')
#Logger.log('next no sooner than {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch'] + updateEvery).strftime('%Y-%m-%d %H:%M:%S')))
#except Exception, e:
# Logger.log("Error: %s" % e)
# raise e
self.lock.releaseLock()
else:
Logger.log('Could not acquire lock')
try:
nameManager = HexNumbersNameManager()
backupProvider = EncryptedBackupProvider(config.storeBackupFolder, config.passphrasePath)
serializer = PickleSerializer()
if len(sys.argv) < 2:
print 'expected at least one command line argument (e.g. backup, restore or ls)'
sys.exit()
if sys.argv[1] == 'backup' :
controller = TreeStoreBackupController(config.dataPath, lock, serializer, nameManager, backupProvider)
controller.runBackup(config.foldersToBackup, config.storeBackupFolder, config.excludePatterns, config.updateBackupEvery)
elif sys.argv[1] == 'ls' :
state = serializer.unserialize(os.path.join(config.dataPath, 'tree.dat'))
for tree in state.trees.values():
for f in tree.getAllFiles():
print f.path
elif sys.argv[1] == 'restore' :
if len(sys.argv) == 5:
pass
else:
print 'you need to specify input and output folder and passphrase file path for restore'
else:
print 'unknown command line option (expected backup or ls)'
except KeyboardInterrupt:
Logger.log('Action aborted by a user')
lock.releaseLock()
def updateFile(self, folder, f, stats):
assert f.dstPath is not None, "missing dstPath for updated file: " + str(f)
Logger.log("Updating or adding file: %s" % f.path)
stats.updatedFilesSize += f.size
stats.updatedFilesCount += 1
self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def addFile(self, folder, f, state, stats):
Logger.log("Updating or adding file: %s" % f.path)
stats.addedFilesSize += f.size
stats.addedFilesCount += 1
f.dstPath = self.nameManager.encodeName(f.path, state)
self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def saveState(self, state):
Logger.log("Serializing state to %s" % self.treeFilePath)
self.serializer.serialize(self.treeFilePath, state)
self.backupProvider.backup(self.treeFilePath, state.treeFilePathEncoded)
def printStats(self, stats):
Logger.log("Added %d files (%.2f MB)" % (stats.addedFilesCount, stats.addedFilesSize/1024/1024))
Logger.log("Updated %d files (%.2f MB)" % (stats.updatedFilesCount, stats.updatedFilesSize/1024/1024))
Logger.log("Removed %d files (%.2f MB)" % (stats.removedFilesCount, stats.removedFilesSize/1024/1024))
def removeFile(self, folder, f, stats):
assert f.dstPath is not None, "missing dstPath for removed file: " + str(f)
Logger.log("Removing file: %s" % f.path)
stats.removedFilesSize += f.size
stats.removedFilesCount += 1
self.backupProvider.remove(f.dstPath)
