Example #1
0
class VirtualFilesystem(AbstractedFS):
    """Represents a virtual filesystem (currently only memory and s3 are supported)
    """
    
    def __init__(self, root, cmd_channel):
        AbstractedFS.__init__(self, root, cmd_channel)
        self.cwd = root
        self.type = cmd_channel.type
        self.s3_bucket = cmd_channel.s3_bucket
        self.aws_access_key = cmd_channel.aws_access_key
        self.aws_secret_key = cmd_channel.aws_secret_key
        self.seperator = cmd_channel.seperator
        self.thread_synchronize = cmd_channel.thread_synchronize
        self.key_sync_timeout = cmd_channel.key_sync_timeout
        if not self.cmd_channel.fs_obj:
            if self.type == "memory":
                self.fs_obj = MemoryFS()
            elif self.type == "s3":
                self.fs_obj = S3FS(bucket=self.bucket, prefix=self.prefix, aws_access_key=self.aws_access_key, aws_secret_key=self.aws_secret_key, separator=self.seperator, thread_synchronize=self.thread_synchronize, key_sync_timeout=self.key_sync_timeout)
            self.cmd_channel.fs_obj = self.fs_obj
        else:
            self.fs_obj = self.cmd_channel.fs_obj
            

    def ftp2fs(self, ftppath):
        return self.ftpnorm(ftppath)

    def fs2ftp(self, fspath):
        return fspath

    def validpath(self, path):
        # validpath was used to check symlinks escaping user home
        # directory; this is no longer necessary.
        return True
    
    def open(self, filename, mode):
            f = self.fs_obj.open(filename, mode)
            f.name=filename
            return f
    
    def mkdir(self, path):
        return self.fs_obj.makedir(path)
        
    def chdir(self, path):
        return self.fs_obj.opendir(path)
    
    def listdir(self,path):
        return self.fs_obj.listdir(path)
    
    def rmdir(self, path):
        return self.fs_obj.removedir(path)
    
    def remove(self, path):
        return self.fs_obj.remove(path)
    
    def rename(self, src, dst):
        return self.fs_obj.rename(src, dst)
    
    def chmod(self, path, mode):
        return True
    
    def readlink(self, path):
        return self.ftp2fs(path)
    
    def isfile(self, path):
        return self.fs_obj.isfile(path)
    
    def islink(self, path):
        return False
    
    def getsize(self, path):
        return self.fs_obj.getsize(path)
    
    def getmtime(self, path):
        return self.fs_obj.getinfo(path)['modified_time']
    
    def realpath(self, path):
        return path
    
    def lexists(self, path):
        return self.fs_obj.exists(path)
    
    def mkstemp(self, suffix='', prefix='', mode='wb'):
        from tempfile import _RandomNameSequence as RandomName
        name = RandomName()
        if suffix != '':
            suffix = 'tmp'
        fname = suffix + name.next()
        return self.fs_obj.open(fname,mode)
class BigFS(FS):

    """A FileSystem that represents a BIG file."""
    
    _meta = { 'virtual' : False,
              'read_only' : True,
              'unicode_paths' : True,
              'case_insensitive_paths' : False,
              'network' : False,                        
             }

    def __init__(self, filename, mode="r", thread_synchronize=True):
        """Create a FS that maps on to a big file.

        :param filename: A (system) path, or a file-like object
        :param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported
        :param thread_synchronize: -- Set to True (default) to enable thread-safety

        """
        super(BigFS, self).__init__(thread_synchronize=thread_synchronize)

        if len(mode) > 1 or mode not in "r":
            raise ValueError("mode must be 'r'")
        self.file_mode = mode
        self.big_path = str(filename)

        self.entries = {}
        try:
            self.bf = open(filename, "rb")
        except IOError:
            raise ResourceNotFoundError(str(filename), msg="BIG file does not exist: %(path)s")

        self._path_fs = MemoryFS()
        if mode in 'ra':
            self._parse_resource_list(self.bf)

    def __str__(self):
        return "<BigFS: %s>" % self.big_path

    def __unicode__(self):
        return unicode(self.__str__())


    def _parse_resource_list(self, g):
        magicWord = g.read(4)
        if magicWord != "BIGF" and magicWord != "BIG4":
            raise ValueError("Magic word of BIG file invalid: " + filename + " " + repr(magicWord))
        header = g.read(12)
        header = unpack(">III", header)
        BIGSize = header[0]
        fileCount = header[1]
        bodyOffset = header[2]
        for i in range(fileCount):
            fileHeader = g.read(8)
            fileHeader = unpack(">II", fileHeader)

            pos = g.tell()
            buf = g.read(4096)
            marker = buf.find("\0")
            if marker == -1:
                raise ValueError("Could not parse filename in BIG file: Too long or invalid file")
            name = buf[:marker]
            # TODO: decode the encoding of name (or normalize the path?)
            isCompressed, uncompressedSize = self.__isCompressed(g, fileHeader[0], fileHeader[1])
            be = BIGEntry(name, fileHeader[0], fileHeader[1], isCompressed, uncompressedSize)
            name = normpath(name)
            self.entries[name] = be
            self._add_resource(name)
            g.seek(pos + marker + 1)

    def __isCompressed(self, g, offset, size):
        g.seek(offset)
        buf = g.read(2)
        magic = unpack(">H", buf)[0]
        if (magic & 0x3EFF) == 0x10FB:
            # it is compressed
            if magic & 0x8000:
                # decompressed size is uint32
                return True, unpack(">I", g.read(4))[0]
            else:
                # use only 3 bytes
                return True, unpack(">I", "\0" + g.read(3))[0]
        return False, size

    def _add_resource(self, path):
        if path.endswith('/'):
            path = path[:-1]
            if path:
                self._path_fs.makedir(path, recursive=True, allow_recreate=True)
        else:
            dirpath, filename = pathsplit(path)
            if dirpath:
                self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True)
            f = self._path_fs.open(path, 'w')
            f.close()


    def close(self):
        """Finalizes the zip file so that it can be read.
        No further operations will work after this method is called."""

        if hasattr(self, 'bf') and self.bf:
            self.bf.close()
            self.bf = _ExceptionProxy()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        path = normpath(relpath(path))        

        if 'r' in mode:
            if self.file_mode not in 'ra':
                raise OperationFailedError("open file", path=path, msg="Big file must be opened for reading ('r') or appending ('a')")
            try:
                return self.entries[path].getfile(self.bf)
            except KeyError:
                raise ResourceNotFoundError(path)

        if 'w' in mode:
            raise OperationFailedError("open file", path=path, msg="Big file cannot be edited ATM")

        raise ValueError("Mode must contain be 'r' or 'w'")

    @synchronize
    def getcontents(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path)
        try:
            contents = self.entries[path].getcontents(self.bf)
        except KeyError:
            raise ResourceNotFoundError(path)
        except RuntimeError:
            raise OperationFailedError("read file", path=path, msg="Big file must be oppened with 'r' or 'a' to read")
        return contents

    def desc(self, path):
        if self.isdir(path):
            return "Dir in big file: %s" % self.big_path
        else:
            return "File in big file: %s" % self.big_path

    def isdir(self, path):
        return self._path_fs.isdir(path)

    def isfile(self, path):
        return self._path_fs.isfile(path)

    def exists(self, path):
        return self._path_fs.exists(path)

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        dirname = normpath(dirname)
        if self.file_mode not in "wa":
            raise OperationFailedError("create directory", path=dirname, msg="Big file must be opened for writing ('w') or appending ('a')")
        if not dirname.endswith('/'):
            dirname += '/'
        self._add_resource(dirname)

    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)

    @synchronize
    def getinfo(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path).lstrip('/')
        info = {'size': 0}
        if path in self.entries:
            be = self.entries[path]
            info['size'] = be.realSize
            info['file_size'] = be.realSize
            info['stored_size'] = be.storedSize
            info['is_compressed'] = be.isCompressed
            info['offset'] = be.offset
            info['internal_filename'] = be.filename
            info['filename'] = path
        return info
Example #3
0
class fs:
    def __init__(self, username="******"):
        self.username = username
        self.__build_default_tree()
        self.NOTFOUND = "Not found"
        self.ALREADYEXISTS = "Already Exists"

    def __create_files_recurse(self, files):
        for file in files:
            if file["type"] == "dir":
                self.mem_fs.makedir(file["name"])
                self.__create_files_recurse(file["children"])
            else:
                data = """��*��̵/"��'g�>����O�����U�\�tc����SAƖ��b���#ں^�+�
                ��,�j��W��=H�a�2�M3��-���2t����5�=��˴�S+���o�G�O�F���B�}����
                ����j4�g���W�%���`3h��0����qss�]�l8�o������l^�VP6d����ƽ��qpD
                o��[n����$V�i�b�{�������XT��dƀݙ���<V6�l���y�N�#)1�+A�xe��#Qd
                y��K��!^��G��@Z�W2L+��M}��S;��qORMX�`��?L*�(��>?�0K6����6G�
                GG�J�&BC�4gO���xb�.��#���"y���k�D�H%��88+B�H�� &l�܋��튰�����
                301���x��mQ������c U{��x_���՞Jt ���y]G��#TԻ�pd{�+)j,wY��4�zN
                .�Y`>W�X)��v��q.L�)K�ry#:��`"L�h"""

                try:
                    data = file["data"]
                except:
                    pass
                self.mem_fs.writetext(file["name"], data)

    def __build_default_tree(self):
        self.mem_fs = MemoryFS()
        dir = os.path.dirname(__file__)
        template_path = os.path.join(dir, "data", "fs_template.json")
        with open(template_path, "r") as raw_template:
            fs_template = json.load(raw_template)
        self.__create_files_recurse(fs_template)
        self.home = f"/home/{self.username}"
        self.pwd = self.home
        self.mem_fs.makedir(self.pwd)
        print(self.mem_fs.listdir("/"))

    def ls(self, path):
        path = self.__format_path(path)
        res = []
        try:
            res = self.mem_fs.listdir(path)
        except fserrors.ResourceNotFound:
            return self.NOTFOUND

        return " ".join(res)

    def mkdir(self, path):
        path = self.__format_path(path)
        try:
            res = self.mem_fs.makedir(path)
        except fserrors.DirectoryExists:
            return self.ALREADYEXISTS
        except fserrors.ResourceNotFound:
            return self.NOTFOUND

    def cd(self, path):
        path = self.__format_path(path)
        try:
            res = self.mem_fs.getinfo(path)
            self.pwd = path
        except fserrors.ResourceNotFound:
            return self.NOTFOUND

    def __format_path(self, path):
        if path == None or len(path) == 0:
            path = self.pwd
        elif path[0] != "~":
            len_rest = self.home
        elif path[0] != "/":
            path = f"{self.pwd}/{path}"
        elif path[0] != ".":
            len_rest = len(path) - 1
            path = f"{self.pwd}/{path[-len_rest:]}"
        return path

    def __str__(self):
        return str(RenderTree(self.root, style=AsciiStyle()))
class SlfFS(FS):
    """
    Implements a read-only file system on top of a SLF-file
    """

    _meta = {
        'thread_safe': False,
        'virtual': False,
        'read_only': True,
        'unicode_paths': False,
        'case_insensitive_paths': False,
        'network': False,
        'atomic.setcontents': False
    }

    def __init__(self, slf_filename):
        super(SlfFS, self).__init__()

        if isinstance(slf_filename, str):
            slf_filename = os.path.expanduser(os.path.expandvars(slf_filename))
            slf_filename = os.path.normpath(os.path.abspath(slf_filename))
            try:
                self.file_name = slf_filename
                self.file = open(slf_filename, 'rb')
            except FileNotFoundError as e:
                raise CreateFailedError(
                    'Slf file not found ({0})'.format(slf_filename),
                    details=e
                )
        else:
            self.file_name = 'file-like'
            self.file = slf_filename

        self.header = SlfHeader.from_bytes(self.file.read(SlfHeader.get_size()))
        self.entries = list(map(self._read_entry, range(self.header['number_of_entries'])))

        self.library_name = self.header['library_name']
        self.library_path = self.header['library_path']
        self.sort = self.header['sort']
        self.version = self.header['version']

        self._path_fs = MemoryFS()
        for e in self.entries:
            path = _get_normalized_filename(e['file_name']).split('/')
            directory = '/'.join(path[:-1]) if len(path) > 2 else '/'

            if self._path_fs.isfile(directory):
                # Sometimes there exists a file that has the same name as a directory
                # Solution: Rename it with a _DIRECTORY_CONFLICT suffix
                self._path_fs.move(directory, directory + DIRECTORY_CONFLICT_SUFFIX)

            if self._path_fs.isdir('/'.join(path)):
                self._path_fs.createfile('/'.join(path) + DIRECTORY_CONFLICT_SUFFIX)
            else:
                self._path_fs.makedir(directory, recursive=True, allow_recreate=True)
                self._path_fs.createfile('/'.join(path))

    def _read_entry(self, index):
        entry_size = SlfEntry.get_size()
        self.file.seek(-entry_size * (self.header['number_of_entries'] - index), os.SEEK_END)
        return SlfEntry.from_bytes(self.file.read(entry_size))

    def __str__(self):
        return '<SlfFS: {0}>'.format(self['library_name'])

    def isfile(self, path):
        return self._path_fs.isfile(path)

    def isdir(self, path):
        return self._path_fs.isdir(path)

    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)

    def open(self, path, mode='r', buffering=-1, encoding='ascii', errors=None, newline=None, line_buffering=False, **kwargs):
        if mode != 'r' and mode != 'rb':
            raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('open'))
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        if self.isdir(path):
            raise ResourceInvalidError(path)
        slf_entry = self._get_slf_entry_for_path(path)

        self.file.seek(slf_entry['offset'], os.SEEK_SET)
        if mode == 'rb':
            return io.BytesIO(self.file.read(slf_entry['length']))
        return io.StringIO(self.file.read(slf_entry['length']).decode(encoding))

    def getinfo(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        if self.isdir(path):
            return {
                'size': 0
            }
        slf_entry = self._get_slf_entry_for_path(path)
        return {
            'size': slf_entry['length'],
            'modified_time': slf_entry['time']
        }

    def makedir(self, path, recursive=False, allow_recreate=False):
        raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('makedir'))

    def remove(self, path):
        raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('remove'))

    def removedir(self, path, recursive=False, force=False):
        raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('removedir'))

    def rename(self, src, dst):
        raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('rename'))

    def _get_slf_entry_for_path(self, path):
        if path.endswith(DIRECTORY_CONFLICT_SUFFIX):
            path = path[:-len(DIRECTORY_CONFLICT_SUFFIX)]
        return next(e for e in self.entries if _get_normalized_filename(e['file_name']) == path)
Example #5
0
class BigFS(FS):
    """A FileSystem that represents a BIG file."""

    _meta = {
        'virtual': False,
        'read_only': True,
        'unicode_paths': True,
        'case_insensitive_paths': False,
        'network': False,
    }

    def __init__(self, filename, mode="r", thread_synchronize=True):
        """Create a FS that maps on to a big file.

        :param filename: A (system) path, or a file-like object
        :param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported
        :param thread_synchronize: -- Set to True (default) to enable thread-safety

        """
        super(BigFS, self).__init__(thread_synchronize=thread_synchronize)

        if len(mode) > 1 or mode not in "r":
            raise ValueError("mode must be 'r'")
        self.file_mode = mode
        self.big_path = str(filename)

        self.entries = {}
        try:
            self.bf = open(filename, "rb")
        except IOError:
            raise ResourceNotFoundError(
                str(filename), msg="BIG file does not exist: %(path)s")

        self._path_fs = MemoryFS()
        if mode in 'ra':
            self._parse_resource_list(self.bf)

    def __str__(self):
        return "<BigFS: %s>" % self.big_path

    def __unicode__(self):
        return unicode(self.__str__())

    def _parse_resource_list(self, g):
        magicWord = g.read(4)
        if magicWord != "BIGF" and magicWord != "BIG4":
            raise ValueError("Magic word of BIG file invalid: " + filename +
                             " " + repr(magicWord))
        header = g.read(12)
        header = unpack(">III", header)
        BIGSize = header[0]
        fileCount = header[1]
        bodyOffset = header[2]
        for i in range(fileCount):
            fileHeader = g.read(8)
            fileHeader = unpack(">II", fileHeader)

            pos = g.tell()
            buf = g.read(4096)
            marker = buf.find("\0")
            if marker == -1:
                raise ValueError(
                    "Could not parse filename in BIG file: Too long or invalid file"
                )
            name = buf[:marker]
            # TODO: decode the encoding of name (or normalize the path?)
            isCompressed, uncompressedSize = self.__isCompressed(
                g, fileHeader[0], fileHeader[1])
            be = BIGEntry(name, fileHeader[0], fileHeader[1], isCompressed,
                          uncompressedSize)
            name = normpath(name)
            self.entries[name] = be
            self._add_resource(name)
            g.seek(pos + marker + 1)

    def __isCompressed(self, g, offset, size):
        g.seek(offset)
        buf = g.read(2)
        magic = unpack(">H", buf)[0]
        if (magic & 0x3EFF) == 0x10FB:
            # it is compressed
            if magic & 0x8000:
                # decompressed size is uint32
                return True, unpack(">I", g.read(4))[0]
            else:
                # use only 3 bytes
                return True, unpack(">I", "\0" + g.read(3))[0]
        return False, size

    def _add_resource(self, path):
        if path.endswith('/'):
            path = path[:-1]
            if path:
                self._path_fs.makedir(path,
                                      recursive=True,
                                      allow_recreate=True)
        else:
            dirpath, filename = pathsplit(path)
            if dirpath:
                self._path_fs.makedir(dirpath,
                                      recursive=True,
                                      allow_recreate=True)
            f = self._path_fs.open(path, 'w')
            f.close()

    def close(self):
        """Finalizes the zip file so that it can be read.
        No further operations will work after this method is called."""

        if hasattr(self, 'bf') and self.bf:
            self.bf.close()
            self.bf = _ExceptionProxy()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        path = normpath(relpath(path))

        if 'r' in mode:
            if self.file_mode not in 'ra':
                raise OperationFailedError(
                    "open file",
                    path=path,
                    msg=
                    "Big file must be opened for reading ('r') or appending ('a')"
                )
            try:
                return self.entries[path].getfile(self.bf)
            except KeyError:
                raise ResourceNotFoundError(path)

        if 'w' in mode:
            raise OperationFailedError("open file",
                                       path=path,
                                       msg="Big file cannot be edited ATM")

        raise ValueError("Mode must contain be 'r' or 'w'")

    @synchronize
    def getcontents(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path)
        try:
            contents = self.entries[path].getcontents(self.bf)
        except KeyError:
            raise ResourceNotFoundError(path)
        except RuntimeError:
            raise OperationFailedError(
                "read file",
                path=path,
                msg="Big file must be oppened with 'r' or 'a' to read")
        return contents

    def desc(self, path):
        if self.isdir(path):
            return "Dir in big file: %s" % self.big_path
        else:
            return "File in big file: %s" % self.big_path

    def isdir(self, path):
        return self._path_fs.isdir(path)

    def isfile(self, path):
        return self._path_fs.isfile(path)

    def exists(self, path):
        return self._path_fs.exists(path)

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        dirname = normpath(dirname)
        if self.file_mode not in "wa":
            raise OperationFailedError(
                "create directory",
                path=dirname,
                msg=
                "Big file must be opened for writing ('w') or appending ('a')")
        if not dirname.endswith('/'):
            dirname += '/'
        self._add_resource(dirname)

    def listdir(self,
                path="/",
                wildcard=None,
                full=False,
                absolute=False,
                dirs_only=False,
                files_only=False):
        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only,
                                     files_only)

    @synchronize
    def getinfo(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path).lstrip('/')
        info = {'size': 0}
        if path in self.entries:
            be = self.entries[path]
            info['size'] = be.realSize
            info['file_size'] = be.realSize
            info['stored_size'] = be.storedSize
            info['is_compressed'] = be.isCompressed
            info['offset'] = be.offset
            info['internal_filename'] = be.filename
            info['filename'] = path
        return info
Example #6
0
from fs.memoryfs import MemoryFS
mem_fs = MemoryFS()

mem_fs.makedirs("HelloMem")
mem_fs.create("./hello.mem")

print(mem_fs.listdir("./"))