class SlfFS(FS): """ Implements a read-only file system on top of a SLF-file """ _meta = { 'thread_safe': False, 'virtual': False, 'read_only': True, 'unicode_paths': False, 'case_insensitive_paths': False, 'network': False, 'atomic.setcontents': False } def __init__(self, slf_filename): super(SlfFS, self).__init__() if isinstance(slf_filename, str): slf_filename = os.path.expanduser(os.path.expandvars(slf_filename)) slf_filename = os.path.normpath(os.path.abspath(slf_filename)) try: self.file_name = slf_filename self.file = open(slf_filename, 'rb') except FileNotFoundError as e: raise CreateFailedError( 'Slf file not found ({0})'.format(slf_filename), details=e ) else: self.file_name = 'file-like' self.file = slf_filename self.header = SlfHeader.from_bytes(self.file.read(SlfHeader.get_size())) self.entries = list(map(self._read_entry, range(self.header['number_of_entries']))) self.library_name = self.header['library_name'] self.library_path = self.header['library_path'] self.sort = self.header['sort'] self.version = self.header['version'] self._path_fs = MemoryFS() for e in self.entries: path = _get_normalized_filename(e['file_name']).split('/') directory = '/'.join(path[:-1]) if len(path) > 2 else '/' if self._path_fs.isfile(directory): # Sometimes there exists a file that has the same name as a directory # Solution: Rename it with a _DIRECTORY_CONFLICT suffix self._path_fs.move(directory, directory + DIRECTORY_CONFLICT_SUFFIX) if self._path_fs.isdir('/'.join(path)): self._path_fs.createfile('/'.join(path) + DIRECTORY_CONFLICT_SUFFIX) else: self._path_fs.makedir(directory, recursive=True, allow_recreate=True) self._path_fs.createfile('/'.join(path)) def _read_entry(self, index): entry_size = SlfEntry.get_size() self.file.seek(-entry_size * (self.header['number_of_entries'] - index), os.SEEK_END) return SlfEntry.from_bytes(self.file.read(entry_size)) def __str__(self): return '<SlfFS: {0}>'.format(self['library_name']) def isfile(self, path): return self._path_fs.isfile(path) def isdir(self, path): return self._path_fs.isdir(path) def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only) def open(self, path, mode='r', buffering=-1, encoding='ascii', errors=None, newline=None, line_buffering=False, **kwargs): if mode != 'r' and mode != 'rb': raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('open')) if not self.exists(path): raise ResourceNotFoundError(path) if self.isdir(path): raise ResourceInvalidError(path) slf_entry = self._get_slf_entry_for_path(path) self.file.seek(slf_entry['offset'], os.SEEK_SET) if mode == 'rb': return io.BytesIO(self.file.read(slf_entry['length'])) return io.StringIO(self.file.read(slf_entry['length']).decode(encoding)) def getinfo(self, path): if not self.exists(path): raise ResourceNotFoundError(path) if self.isdir(path): return { 'size': 0 } slf_entry = self._get_slf_entry_for_path(path) return { 'size': slf_entry['length'], 'modified_time': slf_entry['time'] } def makedir(self, path, recursive=False, allow_recreate=False): raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('makedir')) def remove(self, path): raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('remove')) def removedir(self, path, recursive=False, force=False): raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('removedir')) def rename(self, src, dst): raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('rename')) def _get_slf_entry_for_path(self, path): if path.endswith(DIRECTORY_CONFLICT_SUFFIX): path = path[:-len(DIRECTORY_CONFLICT_SUFFIX)] return next(e for e in self.entries if _get_normalized_filename(e['file_name']) == path)
class BigFS(FS): """A FileSystem that represents a BIG file.""" _meta = { 'virtual' : False, 'read_only' : True, 'unicode_paths' : True, 'case_insensitive_paths' : False, 'network' : False, } def __init__(self, filename, mode="r", thread_synchronize=True): """Create a FS that maps on to a big file. :param filename: A (system) path, or a file-like object :param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported :param thread_synchronize: -- Set to True (default) to enable thread-safety """ super(BigFS, self).__init__(thread_synchronize=thread_synchronize) if len(mode) > 1 or mode not in "r": raise ValueError("mode must be 'r'") self.file_mode = mode self.big_path = str(filename) self.entries = {} try: self.bf = open(filename, "rb") except IOError: raise ResourceNotFoundError(str(filename), msg="BIG file does not exist: %(path)s") self._path_fs = MemoryFS() if mode in 'ra': self._parse_resource_list(self.bf) def __str__(self): return "<BigFS: %s>" % self.big_path def __unicode__(self): return unicode(self.__str__()) def _parse_resource_list(self, g): magicWord = g.read(4) if magicWord != "BIGF" and magicWord != "BIG4": raise ValueError("Magic word of BIG file invalid: " + filename + " " + repr(magicWord)) header = g.read(12) header = unpack(">III", header) BIGSize = header[0] fileCount = header[1] bodyOffset = header[2] for i in range(fileCount): fileHeader = g.read(8) fileHeader = unpack(">II", fileHeader) pos = g.tell() buf = g.read(4096) marker = buf.find("\0") if marker == -1: raise ValueError("Could not parse filename in BIG file: Too long or invalid file") name = buf[:marker] # TODO: decode the encoding of name (or normalize the path?) isCompressed, uncompressedSize = self.__isCompressed(g, fileHeader[0], fileHeader[1]) be = BIGEntry(name, fileHeader[0], fileHeader[1], isCompressed, uncompressedSize) name = normpath(name) self.entries[name] = be self._add_resource(name) g.seek(pos + marker + 1) def __isCompressed(self, g, offset, size): g.seek(offset) buf = g.read(2) magic = unpack(">H", buf)[0] if (magic & 0x3EFF) == 0x10FB: # it is compressed if magic & 0x8000: # decompressed size is uint32 return True, unpack(">I", g.read(4))[0] else: # use only 3 bytes return True, unpack(">I", "\0" + g.read(3))[0] return False, size def _add_resource(self, path): if path.endswith('/'): path = path[:-1] if path: self._path_fs.makedir(path, recursive=True, allow_recreate=True) else: dirpath, filename = pathsplit(path) if dirpath: self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True) f = self._path_fs.open(path, 'w') f.close() def close(self): """Finalizes the zip file so that it can be read. No further operations will work after this method is called.""" if hasattr(self, 'bf') and self.bf: self.bf.close() self.bf = _ExceptionProxy() @synchronize def open(self, path, mode="r", **kwargs): path = normpath(relpath(path)) if 'r' in mode: if self.file_mode not in 'ra': raise OperationFailedError("open file", path=path, msg="Big file must be opened for reading ('r') or appending ('a')") try: return self.entries[path].getfile(self.bf) except KeyError: raise ResourceNotFoundError(path) if 'w' in mode: raise OperationFailedError("open file", path=path, msg="Big file cannot be edited ATM") raise ValueError("Mode must contain be 'r' or 'w'") @synchronize def getcontents(self, path): if not self.exists(path): raise ResourceNotFoundError(path) path = normpath(path) try: contents = self.entries[path].getcontents(self.bf) except KeyError: raise ResourceNotFoundError(path) except RuntimeError: raise OperationFailedError("read file", path=path, msg="Big file must be oppened with 'r' or 'a' to read") return contents def desc(self, path): if self.isdir(path): return "Dir in big file: %s" % self.big_path else: return "File in big file: %s" % self.big_path def isdir(self, path): return self._path_fs.isdir(path) def isfile(self, path): return self._path_fs.isfile(path) def exists(self, path): return self._path_fs.exists(path) @synchronize def makedir(self, dirname, recursive=False, allow_recreate=False): dirname = normpath(dirname) if self.file_mode not in "wa": raise OperationFailedError("create directory", path=dirname, msg="Big file must be opened for writing ('w') or appending ('a')") if not dirname.endswith('/'): dirname += '/' self._add_resource(dirname) def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only) @synchronize def getinfo(self, path): if not self.exists(path): raise ResourceNotFoundError(path) path = normpath(path).lstrip('/') info = {'size': 0} if path in self.entries: be = self.entries[path] info['size'] = be.realSize info['file_size'] = be.realSize info['stored_size'] = be.storedSize info['is_compressed'] = be.isCompressed info['offset'] = be.offset info['internal_filename'] = be.filename info['filename'] = path return info
class BigFS(FS): """A FileSystem that represents a BIG file.""" _meta = { 'virtual': False, 'read_only': True, 'unicode_paths': True, 'case_insensitive_paths': False, 'network': False, } def __init__(self, filename, mode="r", thread_synchronize=True): """Create a FS that maps on to a big file. :param filename: A (system) path, or a file-like object :param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported :param thread_synchronize: -- Set to True (default) to enable thread-safety """ super(BigFS, self).__init__(thread_synchronize=thread_synchronize) if len(mode) > 1 or mode not in "r": raise ValueError("mode must be 'r'") self.file_mode = mode self.big_path = str(filename) self.entries = {} try: self.bf = open(filename, "rb") except IOError: raise ResourceNotFoundError( str(filename), msg="BIG file does not exist: %(path)s") self._path_fs = MemoryFS() if mode in 'ra': self._parse_resource_list(self.bf) def __str__(self): return "<BigFS: %s>" % self.big_path def __unicode__(self): return unicode(self.__str__()) def _parse_resource_list(self, g): magicWord = g.read(4) if magicWord != "BIGF" and magicWord != "BIG4": raise ValueError("Magic word of BIG file invalid: " + filename + " " + repr(magicWord)) header = g.read(12) header = unpack(">III", header) BIGSize = header[0] fileCount = header[1] bodyOffset = header[2] for i in range(fileCount): fileHeader = g.read(8) fileHeader = unpack(">II", fileHeader) pos = g.tell() buf = g.read(4096) marker = buf.find("\0") if marker == -1: raise ValueError( "Could not parse filename in BIG file: Too long or invalid file" ) name = buf[:marker] # TODO: decode the encoding of name (or normalize the path?) isCompressed, uncompressedSize = self.__isCompressed( g, fileHeader[0], fileHeader[1]) be = BIGEntry(name, fileHeader[0], fileHeader[1], isCompressed, uncompressedSize) name = normpath(name) self.entries[name] = be self._add_resource(name) g.seek(pos + marker + 1) def __isCompressed(self, g, offset, size): g.seek(offset) buf = g.read(2) magic = unpack(">H", buf)[0] if (magic & 0x3EFF) == 0x10FB: # it is compressed if magic & 0x8000: # decompressed size is uint32 return True, unpack(">I", g.read(4))[0] else: # use only 3 bytes return True, unpack(">I", "\0" + g.read(3))[0] return False, size def _add_resource(self, path): if path.endswith('/'): path = path[:-1] if path: self._path_fs.makedir(path, recursive=True, allow_recreate=True) else: dirpath, filename = pathsplit(path) if dirpath: self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True) f = self._path_fs.open(path, 'w') f.close() def close(self): """Finalizes the zip file so that it can be read. No further operations will work after this method is called.""" if hasattr(self, 'bf') and self.bf: self.bf.close() self.bf = _ExceptionProxy() @synchronize def open(self, path, mode="r", **kwargs): path = normpath(relpath(path)) if 'r' in mode: if self.file_mode not in 'ra': raise OperationFailedError( "open file", path=path, msg= "Big file must be opened for reading ('r') or appending ('a')" ) try: return self.entries[path].getfile(self.bf) except KeyError: raise ResourceNotFoundError(path) if 'w' in mode: raise OperationFailedError("open file", path=path, msg="Big file cannot be edited ATM") raise ValueError("Mode must contain be 'r' or 'w'") @synchronize def getcontents(self, path): if not self.exists(path): raise ResourceNotFoundError(path) path = normpath(path) try: contents = self.entries[path].getcontents(self.bf) except KeyError: raise ResourceNotFoundError(path) except RuntimeError: raise OperationFailedError( "read file", path=path, msg="Big file must be oppened with 'r' or 'a' to read") return contents def desc(self, path): if self.isdir(path): return "Dir in big file: %s" % self.big_path else: return "File in big file: %s" % self.big_path def isdir(self, path): return self._path_fs.isdir(path) def isfile(self, path): return self._path_fs.isfile(path) def exists(self, path): return self._path_fs.exists(path) @synchronize def makedir(self, dirname, recursive=False, allow_recreate=False): dirname = normpath(dirname) if self.file_mode not in "wa": raise OperationFailedError( "create directory", path=dirname, msg= "Big file must be opened for writing ('w') or appending ('a')") if not dirname.endswith('/'): dirname += '/' self._add_resource(dirname) def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only) @synchronize def getinfo(self, path): if not self.exists(path): raise ResourceNotFoundError(path) path = normpath(path).lstrip('/') info = {'size': 0} if path in self.entries: be = self.entries[path] info['size'] = be.realSize info['file_size'] = be.realSize info['stored_size'] = be.storedSize info['is_compressed'] = be.isCompressed info['offset'] = be.offset info['internal_filename'] = be.filename info['filename'] = path return info