def sync(self):
(serialized, imap_num) = inodemap.inodemap.save_inode_map(get_max_inode())
special_inode = Inode()
special_inode.write(0, serialized)
imap_loc = inodemap.inodemap.lookup(special_inode.id)
segment.segman.update_imap_postion(imap_loc, imap_num)
segment.segman.flush()
async def test_get_inode(filesystem):
# non existed inode eval to False
assert not Inode(MD5)
# existed inode has correct content, ref_count and mime
inode = await Inode.create(gen_content(CONTENT), MIME)
inode = Inode(inode.md5)
await assert_inode(inode)
def find_inode(self, path):
if path == '/':
return self.root_i
dirs = path.split('/')[1:]
with open(self.disk, 'rb') as f:
# TODO: deal with multiple-blocks directories
next_de_pos = (self.root_i.get_blocks()[0] *
self.sb.get_block_size())
while dirs != []:
f.seek(next_de_pos)
data = f.read(DirEntrySize)
de = DirEntry(data)
if de.get_name_length() == 0:
raise NotFound("File %s not found" % path)
if de.get_name() == dirs[0]:
dirs.pop()
inode_pos = (self.sb.get_block_size() *
self.first_bg.get_inode_table() +
(de.get_inode()-1) * InodeSize)
f.seek(inode_pos)
data = f.read(InodeSize)
i = Inode(data)
if dirs == []: # we found it
return i
if i.is_file():
raise NotFound("%s is a file, not a directory" %
de.get_name())
# TODO
next_de_pos = (i.get_blocks()[0] *
self.sb.get_block_size())
next_de_pos += de.get_length()
raise NotFound("Invalid path: %s" % path)
async def lookup(
self,
inode_p: int,
name: bytes,
ctx: pyfuse3.RequestContext = None) -> pyfuse3.EntryAttributes:
"""
Look up a directory entry by name and get its attributes.
This method should return an EntryAttributes instance for the directory entry name in the
directory with inode parent_inode.
If there is no such entry, the method should either return an EntryAttributes instance with
zero st_ino value (in which case the negative lookup will be cached as specified by
entry_timeout), or it should raise FUSEError with an errno of errno.ENOENT (in this case
the negative result will not be cached).
ctx will be a RequestContext instance.
The file system must be able to handle lookups for . and .., no matter if these entries are
returned by readdir or not.
(Successful) execution of this handler increases the lookup count for the returned inode by one.
"""
if name == '.':
info = Inode.by_inode(inode_p)
elif name == '..':
info = Inode.by_inode(inode_p)
info = info.parent_obj()
else:
info = Inode.by_name(inode_p, name.decode())
return self._getattr(info, ctx)
def sync(self):
(serialized,
imap_num) = inodemap.inodemap.save_inode_map(get_max_inode())
special_inode = Inode()
special_inode.write(0, serialized)
imap_loc = inodemap.inodemap.lookup(special_inode.id)
segment.segman.update_imap_postion(imap_loc, imap_num)
segment.segman.flush()
def _getattr(self, info: Inode,
ctx: pyfuse3.RequestContext) -> pyfuse3.EntryAttributes:
if info.inode() == pyfuse3.ROOT_INODE:
is_dir = True
else:
is_dir = info.is_dir()
entry = pyfuse3.EntryAttributes()
entry.st_ino = info.inode()
entry.generation = 0
entry.entry_timeout = 300
entry.attr_timeout = 300
if is_dir:
entry.st_mode = (stat.S_IFDIR | config.MODE_DIR)
else:
entry.st_mode = (stat.S_IFREG | config.MODE_FILE)
entry.st_nlink = 1
entry.st_uid = config.MOUNT_UID
entry.st_gid = config.MOUNT_GID
entry.st_rdev = 0
entry.st_size = info.size()
# entry.st_blksize = config.CHUNKSIZE
if is_dir:
entry.st_blksize = 1_000_000
else:
entry.st_blksize = info.chunk_size()
entry.st_blocks = 1 if is_dir else info.chunks_count()
entry.st_atime_ns = 0
entry.st_mtime_ns = info.mtime() * 1e6
entry.st_ctime_ns = info.ctime() * 1e6
return entry
def stat(self, filename):
inode_no = self.search_dir(filename)
if inode_no is None:
raise Exception("file or directory does not exist")
inode_block = inodemap.inodemap.lookup(inode_no)
inode = Inode(data=segment.segman.block_read(inode_block))
return inode.filesize, inode.isdir
async def test_ref_unref(filesystem):
# ref should increase the ref count
inode = await Inode.create(gen_content(CONTENT), MIME)
await assert_inode(inode)
inode.ref()
await assert_inode(Inode(inode.md5), ref_count=2)
inode.ref()
await assert_inode(Inode(inode.md5), ref_count=3)
# unref should decrease the ref count
inode.unref()
await assert_inode(Inode(inode.md5), ref_count=2)
inode.unref()
await assert_inode(Inode(inode.md5), ref_count=1)
# last unref should remove the inode
inode.unref()
assert not inode
assert not Inode(inode.md5)
def createInodeList(self, inodeTotalNumber, fsimg, offset, size):
inodeList = []
offsetInt = offset
for i in range(inodeTotalNumber):
inodeList.append(Inode())
inodeList[i].createFromBytes(fsimg, offsetInt, size)
offsetInt += size
return inodeList
def getInode (self, ino, vice=None):
if self.inodes.has_key (ino):
inode= self.inodes[ino]
else:
self.debug (1, "not found ino %d" % (ino), 2)
inode= Inode (ino, self, vice=vice, policy=self.policy)
self.inodes[ino]= inode
return inode
def _update_file_handle(self, fh: int) -> Inode:
log.debug('_update_file_handle %s', fh)
self.fh_lock.acquire()
inode = self.file_handles[fh].inode()
inode_info = Inode.by_inode(inode)
self.file_handles[fh] = inode_info
self.fh_lock.release()
return inode_info
def unlink(self, path):
filename = find_filename(path)
parent_inode_no = self.search_dir(find_parent(path))
parent_inode_block = inodemap.inodemap.lookup(parent_inode_no)
parent_inode = Inode(data=segment.segman.block_read(parent_inode_block))
parent_old_size = parent_inode.filesize
parent_dir = Directory(parent_inode_no)
found_entry = False
entries = []
for (name, inode) in parent_dir.enumerate():
if found_entry:
entries.append((name, inode))
if name == filename:
fount = True
position = parent_dir.position - (FILENAMELEN + 4)
for (name, inode) in entries:
parent_inode.write(position, struct.pack("%dsI" % FILENAMELEN, name, inode))
position += FILENAMELEN + 4
parent_inode.filesize = parent_old_size - (FILENAMELEN + 4)
inodemap.inodemap.update_inode(parent_inode_no, parent_inode.serialize())
def create(self, filename, isdir=False):
inode_no = self.search_dir(filename)
if inode_no is not None:
raise Exception("file already exists")
inode = Inode(isdir=isdir)
parent_dir = find_parent(filename)
parent_inode_no = self.search_dir(parent_dir)
if parent_inode_no is None:
raise Exception("parent direntory does not exist")
parent_inode_block = inodemap.inodemap.lookup(parent_inode_no)
parent_inode = Inode(
data=segment.segman.block_read(parent_inode_block))
self.append_entry(parent_inode, find_filename(filename), inode)
if isdir:
return Directory(inode.id)
else:
return File(inode.id)
async def create(self, inode_p: int, name: bytes, mode, flags,
ctx: pyfuse3.RequestContext):
"""
Create a file with permissions mode and open it with flags
ctx will be a RequestContext instance.
The method must return a tuple of the form (fi, attr), where fi is a FileInfo instance
handle like the one returned by open and attr is an EntryAttributes instance with the
attributes of the newly created directory entry.
(Successful) execution of this handler increases the lookup count for the returned inode by one.
"""
inode_info = Inode.by_mkfile(inode_p, name.decode())
fh = self._obtain_file_handle_nofetch(inode_info)
return (pyfuse3.FileInfo(fh=fh), self._getattr(inode_info, ctx))
async def setattr(self, inode: int, attr: pyfuse3.EntryAttributes,
fields: pyfuse3.SetattrFields, fh: int,
ctx: pyfuse3.RequestContext) -> pyfuse3.EntryAttributes:
"""
Change attributes of inode
fields will be an SetattrFields instance that specifies which attributes are
to be updated. attr will be an EntryAttributes instance for inode that contains
the new values for changed attributes, and undefined values for all other attributes.
Most file systems will additionally set the st_ctime_ns attribute to the current
time (to indicate that the inode metadata was changed).
If the syscall that is being processed received a file descriptor argument (like
e.g. ftruncate(2) or fchmod(2)), fh will be the file handle returned by the
corresponding call to the open handler. If the syscall was path based (like e.g.
truncate(2) or chmod(2)), fh will be None.
ctx will be a RequestContext instance.
The method should return an EntryAttributes instance (containing both the changed
and unchanged values).
"""
if fields.update_size:
log.warning('Ignoring update_size, not supported')
if fields.update_mode:
log.warning('Ignoring update_mode, not supported')
if fields.update_uid:
log.warning('Ignoring update_uid, not supported')
if fields.update_gid:
log.warning('Ignoring update_gid, not supported')
if fields.update_atime:
log.warning('Ignoring uptime_atime, not supported')
if fields.update_ctime:
log.warning('Ignoring update_ctime, not supported')
info = Inode.by_inode(inode)
if fields.update_mtime:
info.update_mtime(attr.st_mtime_ns // 1e6)
return self._getattr(info, ctx)
def __init__(self, disk='foo.img', *args, **kw):
fuse.Fuse.__init__(self, *args, **kw)
self.disk = disk
with open(self.disk, 'rb') as f:
f.seek(1024)
data = f.read(SuperBlockSize)
self.sb = SuperBlock(data)
first_group_pos = max(2048, self.sb.get_block_size())
f.seek(first_group_pos)
data = f.read(BlockGroupSize)
self.first_bg = BlockGroup(data)
root_i_pos = (self.sb.get_block_size() *
self.first_bg.get_inode_table() +
InodeSize)
f.seek(root_i_pos)
data = f.read(InodeSize)
self.root_i = Inode(data)
def unlink(self, path):
filename = find_filename(path)
parent_inode_no = self.search_dir(find_parent(path))
parent_inode_block = inodemap.inodemap.lookup(parent_inode_no)
parent_inode = Inode(
data=segment.segman.block_read(parent_inode_block))
parent_old_size = parent_inode.filesize
parent_dir = Directory(parent_inode_no)
found_entry = False
entries = []
for (name, inode) in parent_dir.enumerate():
if found_entry:
entries.append((name, inode))
if name == filename:
fount = True
position = parent_dir.position - (FILENAMELEN + 4)
for (name, inode) in entries:
parent_inode.write(position,
struct.pack("%dsI" % FILENAMELEN, name, inode))
position += (FILENAMELEN + 4)
parent_inode.filesize = parent_old_size - (FILENAMELEN + 4)
inodemap.inodemap.update_inode(parent_inode_no,
parent_inode.serialize())
async def test_set_mime(filesystem):
# mime can be changed afterwards
inode = await Inode.create(gen_content(CONTENT), MIME)
await assert_inode(inode)
inode.mime = MIME2
await assert_inode(Inode(inode.md5), mime=MIME2)
class Ext2(fuse.Fuse):
def __init__(self, disk='foo.img', *args, **kw):
fuse.Fuse.__init__(self, *args, **kw)
self.disk = disk
with open(self.disk, 'rb') as f:
f.seek(1024)
data = f.read(SuperBlockSize)
self.sb = SuperBlock(data)
first_group_pos = max(2048, self.sb.get_block_size())
f.seek(first_group_pos)
data = f.read(BlockGroupSize)
self.first_bg = BlockGroup(data)
root_i_pos = (self.sb.get_block_size() *
self.first_bg.get_inode_table() +
InodeSize)
f.seek(root_i_pos)
data = f.read(InodeSize)
self.root_i = Inode(data)
def find_inode(self, path):
if path == '/':
return self.root_i
dirs = path.split('/')[1:]
with open(self.disk, 'rb') as f:
# TODO: deal with multiple-blocks directories
next_de_pos = (self.root_i.get_blocks()[0] *
self.sb.get_block_size())
while dirs != []:
f.seek(next_de_pos)
data = f.read(DirEntrySize)
de = DirEntry(data)
if de.get_name_length() == 0:
raise NotFound("File %s not found" % path)
if de.get_name() == dirs[0]:
dirs.pop()
inode_pos = (self.sb.get_block_size() *
self.first_bg.get_inode_table() +
(de.get_inode()-1) * InodeSize)
f.seek(inode_pos)
data = f.read(InodeSize)
i = Inode(data)
if dirs == []: # we found it
return i
if i.is_file():
raise NotFound("%s is a file, not a directory" %
de.get_name())
# TODO
next_de_pos = (i.get_blocks()[0] *
self.sb.get_block_size())
next_de_pos += de.get_length()
raise NotFound("Invalid path: %s" % path)
def getattr(self, path):
inode = self.find_inode(path)
stat = fuse.Stat()
stat.st_mode = inode.get_mode()
stat.st_atime = inode.get_atime()
stat.st_mtime = inode.get_mtime()
stat.st_ctime = inode.get_ctime()
stat.st_uid = inode.get_uid()
stat.st_gid = inode.get_gid()
stat.st_size = inode.get_size()
stat.st_nlink = inode.get_links_count()
stat.st_ino = 0 # TODO
stat.st_dev = 0
return stat
def readdir(self, path, offset):
inode = self.find_inode(path)
with open(self.disk, 'rb') as f:
next_de_pos = (inode.get_blocks()[0] *
self.sb.get_block_size())
while True:
f.seek(next_de_pos)
data = f.read(DirEntrySize)
de = DirEntry(data)
if de.get_name_length() == 0:
break
yield fuse.Direntry(de.get_name())
next_de_pos += de.get_length()
return
def mknod(self, path, mode, dev):
return 0
def unlink(self, path):
return 0
def read(self, path, size, offset):
blocks_read = 0
inode = self.find_inode(path)
first_block = offset/self.sb.get_block_size()
data = ''
with open(self.disk, 'rb') as f:
all_blocks = (inode.get_blocks() +
inode.get_indirect_blocks(f, self.sb.get_block_size(), 1))
for block in all_blocks[first_block:]:
if block != 0:
blocks_read += 1
if blocks_read >= MaxBlocksRead:
# we can't give too much data at once
return data
block_pos = (block * self.sb.get_block_size())
f.seek(block_pos)
data += f.read(self.sb.get_block_size())
return data
def write(self, path, buf, offset):
return 0
def release(self, path, flags):
return 0
def open(self, path, flags):
return 0
def truncate(self, path, size):
return 0
def utime(self, path, times):
return 0
def mkdir(self, path, mode):
return 0
def rmdir(self, path):
return 0
def rename(self, pathfrom, pathto):
return 0
def fsync(self, path, isfsyncfile):
return 0
def inode(self):
inode_id = self.meta.get('inode')
return Inode(inode_id) if inode_id else None
from inode import Inode, Tree
from api.functions import splitFile, upload_to_vk, download_from_vk, upload_main_inode
from cache import LRUCache
if __name__ == "__main__":
# Inode test
a = Inode(size=8, blocks={1: range(11200, 11210), 2: range(11200, 11210), 3: range(11200, 11210)})
b = Inode(size=30, blocks={1: range(11200, 11210), 2: range(11200, 11210), 3: range(11200, 11210)})
c = Inode(size=12, blocks={1: range(11200, 11210), 2: range(11200, 11210), 3: range(11200, 11210)})
c1 = Inode(size=15, blocks={1: range(11200, 11210), 2: range(11200, 11210), 3: range(11200, 11210)})
d = Inode(size=2, blocks={1: range(11200, 11210), 2: range(11200, 11210), 3: range(11200, 11210)})
# Tree test
tree = Tree()
#upload_main_inode(tree.marshal())
# tree.mkdir('/home')
# tree.mkdir('/home/horn')
# tree.mkdir('/home/test')
# tree.mkdir('/dom')
# tree.mkdir('/home/test/dir1')
#
# tree.inodes['/']['home']['test'].update({'1.jpg': a})
# tree.inodes['/']['home'].update({'2.jpg': b})
# tree.inodes['/']['home'].update({'3.jpg': c})
# tree.inodes['/']['home'].update({'3.jpg': d})
# print tree
# serialization
#f = open('tree', 'wb')
async def mkdir(self, inode_p: int, name: bytes, _mode,
ctx: pyfuse3.RequestContext) -> pyfuse3.EntryAttributes:
info = Inode.by_mkdir(inode_p, name.decode())
return self._getattr(info, ctx)
def restore(self):
imap_loc = segment.segman.locate_lastest_imap()
iminode = Inode(data=disk.disk.block_read(imap_loc))
imdata = iminode.read(0, 10000000)
set_max_inode(inodemap.inodemap.restore_imap(imdata))
async def getattr(
self,
inode: int,
ctx: pyfuse3.RequestContext = None) -> pyfuse3.EntryAttributes:
info = Inode.by_inode(inode)
return self._getattr(info, ctx)
def restore(self):
imap_loc = segment.segman.locate_lastest_imap()
iminode = Inode(data=disk.disk.block_read(imap_loc))
imdata = iminode.read(0, 10000000)
set_max_inode(inodemap.inodemap.restore_imap(imdata))
def _obtain_file_handle(self, inode: int) -> Tuple[int, Inode]:
inode_info = Inode.by_inode(inode)
return self._obtain_file_handle_nofetch(inode_info), inode_info
def inode(self) -> typing.Optional[Inode]:
inode_id = self.meta.get('inode_md5')
return Inode(inode_id) if inode_id else None
def get_inode(self):
inode_block_no = inodemap.inodemap.lookup(self.inode_no)
#print inode_block_no
inode = Inode(data=segment.segman.block_read(inode_block_no))
return inode