def action_reverse_copy_file(_, params: dict): logger = logging.getLogger(__name__).getChild('action_reverse_copy_file') dest_path = Path(params['from']) orig_path = _checks(params) if 'to' in params or 'path' in params else None orig_stream = params.get('_stream') orig_stat = orig_path.lstat() if orig_path is not None else params['_stat'] preserve_stats = params.get('preserveStats', 'utime') raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean') avoid_copy = False if not params.get('forceCopy', False): logger.debug(f'Checking if the file can be needs to be copied') avoid_copy = not _file_has_changed(orig_stat, dest_path) if dest_path.exists() else False if avoid_copy: logger.debug('File will not be copied') else: logger.debug(f'Copying file {orig_path if orig_path is not None else "*stream*"} to {dest_path}') _write_file( open(orig_path, 'rb', buffering=0) if orig_path is not None else orig_stream, open(dest_path, 'wb', buffering=0), params.get('chunkSize', 1024 * 8), ) if preserve_stats: xattrs = _read_xattrs(orig_path) if orig_path is not None else None _preserve_stats(dest_path, orig_stat, xattrs, preserve_stats)
def action_copy_file(_, params: dict): logger = logging.getLogger(__name__).getChild('action_copy_file') orig_path = Path(params['from']) if params.get('from') is not None else None orig_stream = params.get('_stream') orig_stat = orig_path.lstat() if orig_path is not None else params['_stat'] dest_path = _checks(params) in_path = Path(params['to']) preserve_stats = params.get('preserveStats', 'utime') raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean') avoid_copy = False if params.get('_prev_backup_path') is not None and not params.get('forceCopy', False): logger.debug(f'Checking if the file can be cloned from previous backup') prev_in_path = Path(params['_prev_backup_path']) / in_path avoid_copy = not _file_has_changed(orig_stat, prev_in_path) if prev_in_path.exists() else False if avoid_copy: try: logger.debug(f'Trying to clone {in_path} from previous backup') action_clone_file(None, { 'to': in_path, '_backup_path': params['_backup_path'], 'from': prev_in_path, 'reflink': params.get('reflink', False), 'preserveStats': False, }) except OSError: logger.debug(f'Could not clone {in_path} from previous backup, normal copy will be done') # Probably the clone cannot be done due to hard-link cannot be created avoid_copy = False if not avoid_copy: logger.debug(f'Copying file {orig_path} to {in_path}') action_write_file( open(orig_path, 'rb', buffering=0) if orig_path is not None else orig_stream, { '_backup_path': params['_backup_path'], 'to': in_path, 'chunkSize': params.get('chunkSize', 1024 * 8), }, ) if preserve_stats: xattrs = _read_xattrs(orig_path) if orig_path is not None else None _preserve_stats(dest_path, orig_stat, xattrs, preserve_stats) return dest_path
def action_write_dir(inp: DirEntryGenerator, params: dict): logger = logging.getLogger(__name__).getChild('action_write_dir') if Path(params['path']).is_absolute(): raise ValueError('path cannot be absolute') backup_path = Path(params['_backup_path']) dest_path = Path(params['path']) parent = backup_path / dest_path preserve_stats = params.get('preserveStats', 'utime') parent.mkdir(0o755, parents=params.get('parents', False), exist_ok=True) raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean') logger.debug(f'Writing directory generator to {parent}') for entry in inp: entry_path = parent / entry.path if entry.type == 'dir': logger.debug(f'Creating directory {entry_path}') entry_path.mkdir(0o755, exist_ok=True) elif entry.type == 'symlink': logger.debug( f'Creating symlink {entry_path} pointing to {entry.link_content}' ) os.symlink(entry.link_content, str(entry_path)) elif entry.type == 'file': logger.debug(f'Creating file {entry_path}') action_copy_file( None, { '_stream': entry.stream, '_stat': entry.stats, '_backup_path': params['_backup_path'], '_prev_backup_path': params.get('_prev_backup_path'), 'to': dest_path / entry.path, 'chunkSize': params.get('chunkSize', 1024 * 8), 'reflink': params.get('reflink', False), 'forceCopy': params.get('forceCopy', False), 'preserveStats': False, }) if preserve_stats: logger.debug( f'Modifying stats of file {entry_path} to match the originals') _preserve_stats(entry_path, entry.stats, entry.xattrs, preserve_stats) return parent
def action_clone_file(_, params: dict): logger = logging.getLogger(__name__).getChild('action_clone_file') orig_path = Path(params['from']) dest_path = _checks(params) preserve_stats = params.get('preserveStats', 'utime') raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean') cow_failed = False if params.get('reflink', True): if os.uname().sysname == 'Linux': # Do a Copy on Write clone if possible (only on Linux) # https://stackoverflow.com/questions/52766388/how-can-i-use-the-copy-on-write-of-a-btrfs-from-c-code # https://github.com/coreutils/coreutils/blob/master/src/copy.c#L370 if not dest_path.exists(): dest_path.touch(orig_path.lstat().st_mode) src_fd = os.open(orig_path, flags=os.O_RDONLY) dst_fd = os.open(dest_path, flags=os.O_WRONLY | os.O_TRUNC) try: logger.debug(f'Copying using CoW from {orig_path} to {dest_path}') fcntl.ioctl(dst_fd, 1074041865, src_fd) # FICLONE but hardcoded except OSError: logger.debug('Could not copy using CoW', exc_info=1) cow_failed = True os.close(src_fd) os.close(dst_fd) if cow_failed: os.unlink(dest_path) if preserve_stats: xattrs = _read_xattrs(orig_path) _preserve_stats(dest_path, orig_path.lstat(), xattrs, preserve_stats) else: cow_failed = True if cow_failed or not params.get('reflink', False): # Do a hard-link clone (fallback if CoW fails) logger.debug(f'Creating hardlink from {orig_path} to {dest_path}') os.link(str(orig_path), str(dest_path)) return dest_path
def action_reverse_read_dir(inp, params: dict): logger = logging.getLogger(__name__).getChild('action_reverse_read_dir') if isinstance(params, str): params = {'path': params} root_path = Path(params['path']) preserve_stats = params.get('preserveStats', 'utime') raise_if_type_is_incorrect(preserve_stats, (str, bool), 'preserveStats must be a string or a boolean') root_path.mkdir(0o755, parents=True, exist_ok=True) for entry in inp: entry_path = root_path / entry.path if entry.type == 'dir': logger.debug(f'Creating directory {entry_path}') entry_path.mkdir(0o755, exist_ok=True) elif entry.type == 'symlink': logger.debug( f'Creating symlink {entry_path} pointing to {entry.link_content}' ) os.symlink(entry.link_content, str(entry_path)) elif entry.type == 'file': logger.debug(f'Creating file {entry_path}') action_reverse_copy_file( None, { '_stream': entry.stream, '_stat': entry.stats, '_backup_path': params['_backup_path'], 'from': entry_path, 'chunkSize': params.get('chunkSize', 1024 * 8), 'forceCopy': params.get('forceCopy', False), 'preserveStats': False, }) if preserve_stats: logger.debug( f'Modifying stats of file {entry_path} to match the originals') _preserve_stats(entry_path, entry.stats, entry.xattrs, preserve_stats)