def do_backup_file(self, folder, name): log.trace("do_backup_file", folder, name) path = os.path.join(folder, name) if self.dry_run: print("F - %s" % utils.escape(path)) sys.stdout.flush() return # Add it to the tar try: # What's the encoding on the file? upath = utils.path_to_unicode(path) uname = utils.path_to_unicode(name) ufolder = utils.path_to_unicode(folder) # Due to issues with encoding... I'll open the file myself and pass to tar with open(path, "rb") as f: info = self.tarfile.gettarinfo(arcname=upath, fileobj=f) self.tarfile.addfile(info, f) #self.tarfile.addfile(name=path, recursive=False) mod_time = self.get_file_time_str(path) size = os.path.getsize(path) type = 'F' self.db.fs_saved(upath, mod_time, size, type) self.nfiles += 1 # Save the entry in the LOF self.lof_record(ufolder, uname, "F", mod_time, size) except Exception as e: # If the exception is in the store - we crash and burn # as we cannot save if self.store_thread.error: raise self.store_thread.error # Otherwise log it and keep going... msg = "Unable to backup %s: %s" % (path, str(e)) self.db.save_message(msg) log.warn(msg)
def recover_run(config, db, backup, store, run): wx.Yield() # Grab the LOF tmp_dir = tempfile.mkdtemp() try: # Create encrypted and clear pathnames lof_remote = os.path.join(backup.name, run, const.LOFFile) lof_remote_enc = os.path.join(backup.name, run, const.LOFFile + const.EncryptionSuffix) # Look at the remote site - is it encrypted or not? if store.exists(lof_remote): src = lof_remote encrypted = False elif store.exists(lof_remote_enc): src = lof_remote_enc encrypted = True else: raise Exception(_("Backup '{backup}' run '{run}' is corrupt").format(backup=backup.name, run=run)) # Get the LOF file store.copy_from(src, tmp_dir) lof_file = os.path.join(tmp_dir, const.LOFFile) # Decrypt if required. if encrypted: decrypt_file(config.data_passphrase, lof_file + const.EncryptionSuffix, lof_file) # Now we open and walk through the ZIP file. lof = gzip.GzipFile(lof_file, "rb") folder = "" nfiles = 0 # the config file nfolders = 0 file_sizes = 0 while True: # Get the next line, and remove the CR line = lof.readline() if line is None or line == "": break if line[-1:] == '\n': line = line[:-1] log.debug("Next Line", line) if line == "": # New folder on the next line line = lof.readline() if line is None or line == "": break if line[-1:] == '\n': line = line[:-1] folder = utils.unescape(line) nfolders += 1 else: parts = line.split(",") type = parts[0] name = utils.unescape(parts[1]) path = os.path.join(folder, name) path = utils.path_to_unicode(path) if type == "F" or type == "D": mod_time = parts[2] size = parts[3] db.fs_saved(path, mod_time, size, type) if type == 'F': nfiles += 1 file_sizes += int(size) elif type == 'X': db.fs_deleted(path) else: raise Exception(_("Corrupt type in file list")) wx.Yield() # To calculate the amount of size, we need to see how big the data files are. run_dir = os.path.join(backup.name, run) size = 0 log.debug("Checking files sizes") files = store.list(run_dir) for file in files: size += store.size(os.path.join(run_dir, file)) wx.Yield() finally: shutil.rmtree(tmp_dir) log.trace("Completed recover run: ", size, file_sizes, nfiles, nfolders) return size, file_sizes, nfiles, nfolders
def recursive_backup_folder(self, root): ''' Backup a folder and all its sub-folders. This routine REQUIRES an absolute path. @param folder: ''' log.trace("recursive_backup_folder", root) # Before we interact with the FS - convert to utf-8 root = root.encode('utf-8') if len(root) == 0: raise Exception(_("Backup_folder called on empty folder name")) if root[0] != "/": raise Exception(_("Backup_folder requires absolute paths")) for folder, local_folders, local_files in os.walk(root): # Lets get everything to unicode # local_folders = self.list_to_unicode(local_folders) # local_files = self.list_to_unicode(local_files) log.debug("os.walk", folder, local_folders, local_files) # First: Check if this is specifically excluded if self.check_exclusion(folder): log.info("Excluding Dir:", folder) continue log.info("Backing up folder: %s" % folder) # local_files.sort() # local_folders.sort() # Get the data on this folder from the db db_files = self.db.list_dir(folder) log.debug("Backing up folder", folder) log.debug("local files:", local_files) log.debug("local folders:", local_folders) log.debug("DB files:", db_files) for local_file in local_files: try: local_path = os.path.join(folder, local_file) if self.check_backup(local_path, local_file, db_files): self.do_backup_file(folder, local_file) except StoreFullException as e: log.error(str(e)) raise e except Exception as e: log.warn("Skipping file %s: %s" % (local_file, str(e))) # Convert to unicode for checks below... local_folders = self.list_to_unicode(local_folders) local_files = self.list_to_unicode(local_files) # Have backed up all the local files. Now look for DB files # that exist, but are not local (i.e. they have been deleted) # Make sure we are only looking for 'F' and 'D' (ignore 'X') for db_file in db_files.itervalues(): try: uname = utils.path_to_unicode(db_file.name) if db_file.type in ['D', 'F'] and not uname in local_files and not uname in local_folders: self.do_backup_deleted(folder, db_file.name) except Exception as e: log.warn("Ignoring exception logging deleted file %s: %s" % (db_file.name, e)) for local_folder in local_folders: try: local_path = os.path.join(folder, local_folder) if self.check_backup(local_path, local_folder, db_files): self.do_backup_folder(folder, local_folder) except Exception as e: log.warn("Ignoring exception backing up folder %s: %s" % (local_path, e)) # # At the completion of a folder - we update the DB storage usage if not self.dry_run: self.bytes, self.hash = self.store_thread.get_hash() self.db.update_run_stats(self.bytes, self.nfiles, self.nfolders, self.backup.include_packages, self.hash)
def list_to_unicode(self, l): return [utils.path_to_unicode(p) for p in l]