def __init__(self, repo_path, autocommit=True): self.eg = EasyGit.open_repo(repo_path) features = FeatureBlob(self.eg.root['features']) assert features.get('inode_format', None) == 'treetree' assert features.get('inode_index_format', None) == 'treetree' self.autocommit = autocommit log.debug('Loaded storage, autocommit=%r, HEAD=%r', autocommit, self.eg.get_head_id()) self._inode_cache = {} self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it')
def convert_fs_to_treetree_inode_index(eg): """ Convert a filesystem from the "inode index as as flat list" format to the "inode index as treetree" format. """ inode_index_raw = eg.root['inodes'] inode_index_tt = TreeTree(inode_index_raw, prefix='it') all_inode_names = list(inode_index_raw.keys()) largest_number = -1 for inode_name in all_inode_names: upgrade_log.debug('Moving inode %r to treetree', inode_name) inode_index_tt.clone(inode_index_raw[inode_name], inode_name[1:]) del inode_index_raw[inode_name] number = int(inode_name[1:]) largest_number = max(largest_number, number) FeatureBlob(eg.root['features'])['next_inode_number'] = largest_number + 1
class GitStorage(object): commit_author = "Spaghetti User <*****@*****.**>" @classmethod def create(cls, repo_path): if not os.path.isdir(repo_path): os.mkdir(repo_path) eg = EasyGit.new_repo(repo_path, bare=True) inodes = eg.root.new_tree('inodes') root_ls = eg.root.new_blob('root.ls') root_sub = eg.root.new_tree('root.sub') features_blob = eg.root.new_blob('features') features_blob.data = '{}' features = FeatureBlob(features_blob) features['next_inode_number'] = 1 features['inode_index_format'] = 'treetree' features['inode_format'] = 'treetree' eg.commit(cls.commit_author, 'Created empty filesystem') return cls(repo_path) def __init__(self, repo_path, autocommit=True): self.eg = EasyGit.open_repo(repo_path) features = FeatureBlob(self.eg.root['features']) assert features.get('inode_format', None) == 'treetree' assert features.get('inode_index_format', None) == 'treetree' self.autocommit = autocommit log.debug('Loaded storage, autocommit=%r, HEAD=%r', autocommit, self.eg.get_head_id()) self._inode_cache = {} self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it') def get_root(self): commit_tree = self.eg.root root_ls = commit_tree['root.ls'] root_sub = commit_tree['root.sub'] root = StorageDir('root', root_ls, root_sub, '/', self, None) root.path = '/' return root def get_inode(self, name): if name in self._inode_cache: inode = self._inode_cache[name]() if inode is None: del self._inode_cache[name] else: return inode inode_tree = self._inodes_tt[name[1:]] inode = StorageInode(name, inode_tree, self) self._inode_cache[name] = weakref.ref(inode) return inode def create_inode(self): features = FeatureBlob(self.eg.root['features']) next_inode_number = features['next_inode_number'] features['next_inode_number'] = next_inode_number + 1 inode_name = 'i%d' % next_inode_number inode_tree = self._inodes_tt.new_tree(inode_name[1:]) inode_tree.new_blob('meta').data = StorageInode.default_meta return self.get_inode(inode_name) def _remove_inode(self, name): if name in self._inode_cache: del self._inode_cache[name] def _autocommit(self): if self.autocommit: self.commit("Auto commit") def commit(self, message=None, amend=False, head_id=None, branch='master'): log.info('Committing') if head_id is None: head_id = self.eg.get_head_id(branch) if amend: git = self.eg.git prev_commit = git.commit(head_id) parents = prev_commit.parents if message is None: message = prev_commit.message else: parents = [head_id] assert message is not None self.eg.commit(self.commit_author, message, parents, branch=branch)
def __init__(self, name, tree, storage): self.name = name self.tree = tree self.storage = storage self.tt = TreeTree(tree, prefix='bt') log.debug('Loaded inode %r', name)
class StorageInode(object): blocksize = 64 * 1024 # 64 KB #blocksize = 1024*1024 # 1024 KB default_meta = ('mode: 0100644\n' 'nlink: 1\n' 'uid: 0\n' 'gid: 0\n' 'size: 0\n') int_meta = ('nlink', 'uid', 'gid', 'size') oct_meta = ('mode', ) def __init__(self, name, tree, storage): self.name = name self.tree = tree self.storage = storage self.tt = TreeTree(tree, prefix='bt') log.debug('Loaded inode %r', name) def _read_meta(self): try: meta_blob = self.tree['meta'] except KeyError: meta_raw = self.default_meta else: meta_raw = meta_blob.data return dict( line.split(': ', 1) for line in meta_raw.strip().split('\n')) def _write_meta(self, meta_data): meta_raw = ''.join('%s: %s\n' % (key, value) for key, value in sorted(meta_data.items())) self.tree.new_blob('meta').data = meta_raw self.storage._autocommit() def __getitem__(self, key): value = self._read_meta()[key] if key in self.oct_meta: value = int(value, base=8) elif key in self.int_meta: value = int(value) return value def __setitem__(self, key, value): if key in self.oct_meta: value = '0%o' % value elif key in self.int_meta: value = '%d' % value else: raise NotImplementedError meta_data = self._read_meta() meta_data[key] = value self._write_meta(meta_data) def read_block(self, n): block_name = str(n) log.debug('Reading block %r of inode %r', block_name, self.name) try: block = self.tt[block_name] except KeyError: return '' else: return block.data def write_block(self, n, data): block_name = str(n) log.debug('Writing block %r of inode %r', block_name, self.name) try: block = self.tt[block_name] except KeyError: block = self.tt.new_blob(block_name) block.data = data self.storage._autocommit() def delete_block(self, n): block_name = str(n) log.debug('Removing block %r of inode %r', block_name, self.name) del self.tt[block_name] self.storage._autocommit() def read_data(self, offset, length): end = offset + length eof = self['size'] if end > eof: end = eof length = end - offset if length <= 0: return '' first_block = offset / self.blocksize last_block = end / self.blocksize output = StringIO() for n_block in range(first_block, last_block + 1): block_offset = n_block * self.blocksize fragment_offset = 0 if n_block == first_block: fragment_offset = offset - block_offset fragment_end = self.blocksize if n_block == last_block: fragment_end = end - block_offset block_data = self.read_block(n_block) fragment = block_data[fragment_offset:fragment_end] assert len(fragment) == fragment_end - fragment_offset output.write(fragment) output = output.getvalue() assert len(output) == length return output def write_data(self, data, offset): current_size = self['size'] if current_size < offset: self.truncate(offset) log.info('Inode %s writing %d bytes at offset %d', repr(self.name), len(data), offset) end = offset + len(data) first_block = offset / self.blocksize last_block = end / self.blocksize for n_block in range(first_block, last_block + 1): block_offset = n_block * self.blocksize insert_offset = 0 if n_block == first_block: insert_offset = offset - block_offset insert_end = self.blocksize if n_block == last_block: insert_end = end - block_offset data_start = block_offset + insert_offset - offset data_end = block_offset + insert_end - offset log.debug( 'Updating inode %d between (%d, %d) ' 'with data slice between (%d, %d)', n_block, insert_offset, insert_end, data_start, data_end) current_data = self.read_block(n_block) datafile = StringIO() datafile.write(current_data) datafile.seek(insert_offset) datafile.write(data[data_start:data_end]) self.write_block(n_block, datafile.getvalue()) if end > current_size: self['size'] = end def truncate(self, new_size): log.info("Truncating inode %s, new size %d", repr(self.name), new_size) current_size = self['size'] if current_size < new_size: # TODO: avoid creating one big string self.write_data('\0' * (new_size - current_size), current_size) elif current_size > new_size: first_block = new_size / self.blocksize last_block = current_size / self.blocksize truncate_offset = new_size % self.blocksize for n_block in range(first_block, last_block + 1): if n_block == first_block and truncate_offset > 0: old_data = self.read_block(n_block) self.write_block(n_block, old_data[:truncate_offset]) else: self.delete_block(n_block) self['size'] = new_size def unlink(self): log.info('Unlinking inode %r', self.name) nlink = self['nlink'] - 1 if nlink > 0: log.info('Links remaining for inode %r: %d', self.name, nlink) self['nlink'] = nlink else: log.info('Links remaining for inode %r: 0; removing.', self.name) self.storage._remove_inode(self.name) self.tree.remove() self.storage._autocommit()
class StorageInode(object): blocksize = 64*1024 # 64 KB #blocksize = 1024*1024 # 1024 KB default_meta = ('mode: 0100644\n' 'nlink: 1\n' 'uid: 0\n' 'gid: 0\n' 'size: 0\n') int_meta = ('nlink', 'uid', 'gid', 'size') oct_meta = ('mode',) def __init__(self, name, tree, storage): self.name = name self.tree = tree self.storage = storage self.tt = TreeTree(tree, prefix='bt') log.debug('Loaded inode %r', name) def _read_meta(self): try: meta_blob = self.tree['meta'] except KeyError: meta_raw = self.default_meta else: meta_raw = meta_blob.data return dict(line.split(': ', 1) for line in meta_raw.strip().split('\n')) def _write_meta(self, meta_data): meta_raw = ''.join('%s: %s\n' % (key, value) for key, value in sorted(meta_data.items())) self.tree.new_blob('meta').data = meta_raw self.storage._autocommit() def __getitem__(self, key): value = self._read_meta()[key] if key in self.oct_meta: value = int(value, base=8) elif key in self.int_meta: value = int(value) return value def __setitem__(self, key, value): if key in self.oct_meta: value = '0%o' % value elif key in self.int_meta: value = '%d' % value else: raise NotImplementedError meta_data = self._read_meta() meta_data[key] = value self._write_meta(meta_data) def read_block(self, n): block_name = str(n) log.debug('Reading block %r of inode %r', block_name, self.name) try: block = self.tt[block_name] except KeyError: return '' else: return block.data def write_block(self, n, data): block_name = str(n) log.debug('Writing block %r of inode %r', block_name, self.name) try: block = self.tt[block_name] except KeyError: block = self.tt.new_blob(block_name) block.data = data self.storage._autocommit() def delete_block(self, n): block_name = str(n) log.debug('Removing block %r of inode %r', block_name, self.name) del self.tt[block_name] self.storage._autocommit() def read_data(self, offset, length): end = offset + length eof = self['size'] if end > eof: end = eof length = end - offset if length <= 0: return '' first_block = offset / self.blocksize last_block = end / self.blocksize output = StringIO() for n_block in range(first_block, last_block+1): block_offset = n_block * self.blocksize fragment_offset = 0 if n_block == first_block: fragment_offset = offset - block_offset fragment_end = self.blocksize if n_block == last_block: fragment_end = end - block_offset block_data = self.read_block(n_block) fragment = block_data[fragment_offset:fragment_end] assert len(fragment) == fragment_end - fragment_offset output.write(fragment) output = output.getvalue() assert len(output) == length return output def write_data(self, data, offset): current_size = self['size'] if current_size < offset: self.truncate(offset) log.info('Inode %s writing %d bytes at offset %d', repr(self.name), len(data), offset) end = offset + len(data) first_block = offset / self.blocksize last_block = end / self.blocksize for n_block in range(first_block, last_block+1): block_offset = n_block * self.blocksize insert_offset = 0 if n_block == first_block: insert_offset = offset - block_offset insert_end = self.blocksize if n_block == last_block: insert_end = end - block_offset data_start = block_offset + insert_offset - offset data_end = block_offset + insert_end - offset log.debug('Updating inode %d between (%d, %d) ' 'with data slice between (%d, %d)', n_block, insert_offset, insert_end, data_start, data_end) current_data = self.read_block(n_block) datafile = StringIO() datafile.write(current_data) datafile.seek(insert_offset) datafile.write(data[data_start:data_end]) self.write_block(n_block, datafile.getvalue()) if end > current_size: self['size'] = end def truncate(self, new_size): log.info("Truncating inode %s, new size %d", repr(self.name), new_size) current_size = self['size'] if current_size < new_size: # TODO: avoid creating one big string self.write_data('\0' * (new_size - current_size), current_size) elif current_size > new_size: first_block = new_size / self.blocksize last_block = current_size / self.blocksize truncate_offset = new_size % self.blocksize for n_block in range(first_block, last_block+1): if n_block == first_block and truncate_offset > 0: old_data = self.read_block(n_block) self.write_block(n_block, old_data[:truncate_offset]) else: self.delete_block(n_block) self['size'] = new_size def unlink(self): log.info('Unlinking inode %r', self.name) nlink = self['nlink'] - 1 if nlink > 0: log.info('Links remaining for inode %r: %d', self.name, nlink) self['nlink'] = nlink else: log.info('Links remaining for inode %r: 0; removing.', self.name) self.storage._remove_inode(self.name) self.tree.remove() self.storage._autocommit()