def do_backup_file(self, folder, name):
log.trace("do_backup_file", folder, name)
path = os.path.join(folder, name)
if self.dry_run:
print("F - %s" % utils.escape(path))
sys.stdout.flush()
return
# Add it to the tar
try:
# What's the encoding on the file?
upath = utils.path_to_unicode(path)
uname = utils.path_to_unicode(name)
ufolder = utils.path_to_unicode(folder)
# Due to issues with encoding... I'll open the file myself and pass to tar
with open(path, "rb") as f:
info = self.tarfile.gettarinfo(arcname=upath, fileobj=f)
self.tarfile.addfile(info, f)
#self.tarfile.addfile(name=path, recursive=False)
mod_time = self.get_file_time_str(path)
size = os.path.getsize(path)
type = 'F'
self.db.fs_saved(upath, mod_time, size, type)
self.nfiles += 1
# Save the entry in the LOF
self.lof_record(ufolder, uname, "F", mod_time, size)
except Exception as e:
# If the exception is in the store - we crash and burn
# as we cannot save
if self.store_thread.error:
raise self.store_thread.error
# Otherwise log it and keep going...
msg = "Unable to backup %s: %s" % (path, str(e))
self.db.save_message(msg)
log.warn(msg)
def recover_run(config, db, backup, store, run):
wx.Yield()
# Grab the LOF
tmp_dir = tempfile.mkdtemp()
try:
# Create encrypted and clear pathnames
lof_remote = os.path.join(backup.name, run, const.LOFFile)
lof_remote_enc = os.path.join(backup.name, run, const.LOFFile + const.EncryptionSuffix)
# Look at the remote site - is it encrypted or not?
if store.exists(lof_remote):
src = lof_remote
encrypted = False
elif store.exists(lof_remote_enc):
src = lof_remote_enc
encrypted = True
else:
raise Exception(_("Backup '{backup}' run '{run}' is corrupt").format(backup=backup.name, run=run))
# Get the LOF file
store.copy_from(src, tmp_dir)
lof_file = os.path.join(tmp_dir, const.LOFFile)
# Decrypt if required.
if encrypted:
decrypt_file(config.data_passphrase, lof_file + const.EncryptionSuffix, lof_file)
# Now we open and walk through the ZIP file.
lof = gzip.GzipFile(lof_file, "rb")
folder = ""
nfiles = 0 # the config file
nfolders = 0
file_sizes = 0
while True:
# Get the next line, and remove the CR
line = lof.readline()
if line is None or line == "":
break
if line[-1:] == '\n':
line = line[:-1]
log.debug("Next Line", line)
if line == "":
# New folder on the next line
line = lof.readline()
if line is None or line == "":
break
if line[-1:] == '\n':
line = line[:-1]
folder = utils.unescape(line)
nfolders += 1
else:
parts = line.split(",")
type = parts[0]
name = utils.unescape(parts[1])
path = os.path.join(folder, name)
path = utils.path_to_unicode(path)
if type == "F" or type == "D":
mod_time = parts[2]
size = parts[3]
db.fs_saved(path, mod_time, size, type)
if type == 'F':
nfiles += 1
file_sizes += int(size)
elif type == 'X':
db.fs_deleted(path)
else:
raise Exception(_("Corrupt type in file list"))
wx.Yield()
# To calculate the amount of size, we need to see how big the data files are.
run_dir = os.path.join(backup.name, run)
size = 0
log.debug("Checking files sizes")
files = store.list(run_dir)
for file in files:
size += store.size(os.path.join(run_dir, file))
wx.Yield()
finally:
shutil.rmtree(tmp_dir)
log.trace("Completed recover run: ", size, file_sizes, nfiles, nfolders)
return size, file_sizes, nfiles, nfolders
def recursive_backup_folder(self, root):
'''
Backup a folder and all its sub-folders.
This routine REQUIRES an absolute path.
@param folder:
'''
log.trace("recursive_backup_folder", root)
# Before we interact with the FS - convert to utf-8
root = root.encode('utf-8')
if len(root) == 0:
raise Exception(_("Backup_folder called on empty folder name"))
if root[0] != "/":
raise Exception(_("Backup_folder requires absolute paths"))
for folder, local_folders, local_files in os.walk(root):
# Lets get everything to unicode
# local_folders = self.list_to_unicode(local_folders)
# local_files = self.list_to_unicode(local_files)
log.debug("os.walk", folder, local_folders, local_files)
# First: Check if this is specifically excluded
if self.check_exclusion(folder):
log.info("Excluding Dir:", folder)
continue
log.info("Backing up folder: %s" % folder)
# local_files.sort()
# local_folders.sort()
# Get the data on this folder from the db
db_files = self.db.list_dir(folder)
log.debug("Backing up folder", folder)
log.debug("local files:", local_files)
log.debug("local folders:", local_folders)
log.debug("DB files:", db_files)
for local_file in local_files:
try:
local_path = os.path.join(folder, local_file)
if self.check_backup(local_path, local_file, db_files):
self.do_backup_file(folder, local_file)
except StoreFullException as e:
log.error(str(e))
raise e
except Exception as e:
log.warn("Skipping file %s: %s" % (local_file, str(e)))
# Convert to unicode for checks below...
local_folders = self.list_to_unicode(local_folders)
local_files = self.list_to_unicode(local_files)
# Have backed up all the local files. Now look for DB files
# that exist, but are not local (i.e. they have been deleted)
# Make sure we are only looking for 'F' and 'D' (ignore 'X')
for db_file in db_files.itervalues():
try:
uname = utils.path_to_unicode(db_file.name)
if db_file.type in ['D', 'F'] and not uname in local_files and not uname in local_folders:
self.do_backup_deleted(folder, db_file.name)
except Exception as e:
log.warn("Ignoring exception logging deleted file %s: %s" % (db_file.name, e))
for local_folder in local_folders:
try:
local_path = os.path.join(folder, local_folder)
if self.check_backup(local_path, local_folder, db_files):
self.do_backup_folder(folder, local_folder)
except Exception as e:
log.warn("Ignoring exception backing up folder %s: %s" % (local_path, e))
# # At the completion of a folder - we update the DB storage usage
if not self.dry_run:
self.bytes, self.hash = self.store_thread.get_hash()
self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)
def list_to_unicode(self, l):
return [utils.path_to_unicode(p) for p in l]
