def __init__(self, repo_path, autocommit=True):
self.eg = EasyGit.open_repo(repo_path)
features = FeatureBlob(self.eg.root['features'])
assert features.get('inode_format', None) == 'treetree'
assert features.get('inode_index_format', None) == 'treetree'
self.autocommit = autocommit
log.debug('Loaded storage, autocommit=%r, HEAD=%r', autocommit,
self.eg.get_head_id())
self._inode_cache = {}
self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it')
def __init__(self, repo_path, autocommit=True):
self.eg = EasyGit.open_repo(repo_path)
features = FeatureBlob(self.eg.root['features'])
assert features.get('inode_format', None) == 'treetree'
assert features.get('inode_index_format', None) == 'treetree'
self.autocommit = autocommit
log.debug('Loaded storage, autocommit=%r, HEAD=%r',
autocommit, self.eg.get_head_id())
self._inode_cache = {}
self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it')
def convert_fs_to_treetree_inode_index(eg):
"""
Convert a filesystem from the "inode index as as flat list" format to the
"inode index as treetree" format.
"""
inode_index_raw = eg.root['inodes']
inode_index_tt = TreeTree(inode_index_raw, prefix='it')
all_inode_names = list(inode_index_raw.keys())
largest_number = -1
for inode_name in all_inode_names:
upgrade_log.debug('Moving inode %r to treetree', inode_name)
inode_index_tt.clone(inode_index_raw[inode_name], inode_name[1:])
del inode_index_raw[inode_name]
number = int(inode_name[1:])
largest_number = max(largest_number, number)
FeatureBlob(eg.root['features'])['next_inode_number'] = largest_number + 1
def convert_fs_to_treetree_inode_index(eg):
"""
Convert a filesystem from the "inode index as as flat list" format to the
"inode index as treetree" format.
"""
inode_index_raw = eg.root['inodes']
inode_index_tt = TreeTree(inode_index_raw, prefix='it')
all_inode_names = list(inode_index_raw.keys())
largest_number = -1
for inode_name in all_inode_names:
upgrade_log.debug('Moving inode %r to treetree', inode_name)
inode_index_tt.clone(inode_index_raw[inode_name], inode_name[1:])
del inode_index_raw[inode_name]
number = int(inode_name[1:])
largest_number = max(largest_number, number)
FeatureBlob(eg.root['features'])['next_inode_number'] = largest_number + 1
class GitStorage(object):
commit_author = "Spaghetti User <*****@*****.**>"
@classmethod
def create(cls, repo_path):
if not os.path.isdir(repo_path):
os.mkdir(repo_path)
eg = EasyGit.new_repo(repo_path, bare=True)
inodes = eg.root.new_tree('inodes')
root_ls = eg.root.new_blob('root.ls')
root_sub = eg.root.new_tree('root.sub')
features_blob = eg.root.new_blob('features')
features_blob.data = '{}'
features = FeatureBlob(features_blob)
features['next_inode_number'] = 1
features['inode_index_format'] = 'treetree'
features['inode_format'] = 'treetree'
eg.commit(cls.commit_author, 'Created empty filesystem')
return cls(repo_path)
def __init__(self, repo_path, autocommit=True):
self.eg = EasyGit.open_repo(repo_path)
features = FeatureBlob(self.eg.root['features'])
assert features.get('inode_format', None) == 'treetree'
assert features.get('inode_index_format', None) == 'treetree'
self.autocommit = autocommit
log.debug('Loaded storage, autocommit=%r, HEAD=%r', autocommit,
self.eg.get_head_id())
self._inode_cache = {}
self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it')
def get_root(self):
commit_tree = self.eg.root
root_ls = commit_tree['root.ls']
root_sub = commit_tree['root.sub']
root = StorageDir('root', root_ls, root_sub, '/', self, None)
root.path = '/'
return root
def get_inode(self, name):
if name in self._inode_cache:
inode = self._inode_cache[name]()
if inode is None:
del self._inode_cache[name]
else:
return inode
inode_tree = self._inodes_tt[name[1:]]
inode = StorageInode(name, inode_tree, self)
self._inode_cache[name] = weakref.ref(inode)
return inode
def create_inode(self):
features = FeatureBlob(self.eg.root['features'])
next_inode_number = features['next_inode_number']
features['next_inode_number'] = next_inode_number + 1
inode_name = 'i%d' % next_inode_number
inode_tree = self._inodes_tt.new_tree(inode_name[1:])
inode_tree.new_blob('meta').data = StorageInode.default_meta
return self.get_inode(inode_name)
def _remove_inode(self, name):
if name in self._inode_cache:
del self._inode_cache[name]
def _autocommit(self):
if self.autocommit:
self.commit("Auto commit")
def commit(self, message=None, amend=False, head_id=None, branch='master'):
log.info('Committing')
if head_id is None:
head_id = self.eg.get_head_id(branch)
if amend:
git = self.eg.git
prev_commit = git.commit(head_id)
parents = prev_commit.parents
if message is None:
message = prev_commit.message
else:
parents = [head_id]
assert message is not None
self.eg.commit(self.commit_author, message, parents, branch=branch)
def __init__(self, name, tree, storage):
self.name = name
self.tree = tree
self.storage = storage
self.tt = TreeTree(tree, prefix='bt')
log.debug('Loaded inode %r', name)
class StorageInode(object):
blocksize = 64 * 1024 # 64 KB
#blocksize = 1024*1024 # 1024 KB
default_meta = ('mode: 0100644\n'
'nlink: 1\n'
'uid: 0\n'
'gid: 0\n'
'size: 0\n')
int_meta = ('nlink', 'uid', 'gid', 'size')
oct_meta = ('mode', )
def __init__(self, name, tree, storage):
self.name = name
self.tree = tree
self.storage = storage
self.tt = TreeTree(tree, prefix='bt')
log.debug('Loaded inode %r', name)
def _read_meta(self):
try:
meta_blob = self.tree['meta']
except KeyError:
meta_raw = self.default_meta
else:
meta_raw = meta_blob.data
return dict(
line.split(': ', 1) for line in meta_raw.strip().split('\n'))
def _write_meta(self, meta_data):
meta_raw = ''.join('%s: %s\n' % (key, value)
for key, value in sorted(meta_data.items()))
self.tree.new_blob('meta').data = meta_raw
self.storage._autocommit()
def __getitem__(self, key):
value = self._read_meta()[key]
if key in self.oct_meta:
value = int(value, base=8)
elif key in self.int_meta:
value = int(value)
return value
def __setitem__(self, key, value):
if key in self.oct_meta:
value = '0%o' % value
elif key in self.int_meta:
value = '%d' % value
else:
raise NotImplementedError
meta_data = self._read_meta()
meta_data[key] = value
self._write_meta(meta_data)
def read_block(self, n):
block_name = str(n)
log.debug('Reading block %r of inode %r', block_name, self.name)
try:
block = self.tt[block_name]
except KeyError:
return ''
else:
return block.data
def write_block(self, n, data):
block_name = str(n)
log.debug('Writing block %r of inode %r', block_name, self.name)
try:
block = self.tt[block_name]
except KeyError:
block = self.tt.new_blob(block_name)
block.data = data
self.storage._autocommit()
def delete_block(self, n):
block_name = str(n)
log.debug('Removing block %r of inode %r', block_name, self.name)
del self.tt[block_name]
self.storage._autocommit()
def read_data(self, offset, length):
end = offset + length
eof = self['size']
if end > eof:
end = eof
length = end - offset
if length <= 0:
return ''
first_block = offset / self.blocksize
last_block = end / self.blocksize
output = StringIO()
for n_block in range(first_block, last_block + 1):
block_offset = n_block * self.blocksize
fragment_offset = 0
if n_block == first_block:
fragment_offset = offset - block_offset
fragment_end = self.blocksize
if n_block == last_block:
fragment_end = end - block_offset
block_data = self.read_block(n_block)
fragment = block_data[fragment_offset:fragment_end]
assert len(fragment) == fragment_end - fragment_offset
output.write(fragment)
output = output.getvalue()
assert len(output) == length
return output
def write_data(self, data, offset):
current_size = self['size']
if current_size < offset:
self.truncate(offset)
log.info('Inode %s writing %d bytes at offset %d', repr(self.name),
len(data), offset)
end = offset + len(data)
first_block = offset / self.blocksize
last_block = end / self.blocksize
for n_block in range(first_block, last_block + 1):
block_offset = n_block * self.blocksize
insert_offset = 0
if n_block == first_block:
insert_offset = offset - block_offset
insert_end = self.blocksize
if n_block == last_block:
insert_end = end - block_offset
data_start = block_offset + insert_offset - offset
data_end = block_offset + insert_end - offset
log.debug(
'Updating inode %d between (%d, %d) '
'with data slice between (%d, %d)', n_block, insert_offset,
insert_end, data_start, data_end)
current_data = self.read_block(n_block)
datafile = StringIO()
datafile.write(current_data)
datafile.seek(insert_offset)
datafile.write(data[data_start:data_end])
self.write_block(n_block, datafile.getvalue())
if end > current_size:
self['size'] = end
def truncate(self, new_size):
log.info("Truncating inode %s, new size %d", repr(self.name), new_size)
current_size = self['size']
if current_size < new_size:
# TODO: avoid creating one big string
self.write_data('\0' * (new_size - current_size), current_size)
elif current_size > new_size:
first_block = new_size / self.blocksize
last_block = current_size / self.blocksize
truncate_offset = new_size % self.blocksize
for n_block in range(first_block, last_block + 1):
if n_block == first_block and truncate_offset > 0:
old_data = self.read_block(n_block)
self.write_block(n_block, old_data[:truncate_offset])
else:
self.delete_block(n_block)
self['size'] = new_size
def unlink(self):
log.info('Unlinking inode %r', self.name)
nlink = self['nlink'] - 1
if nlink > 0:
log.info('Links remaining for inode %r: %d', self.name, nlink)
self['nlink'] = nlink
else:
log.info('Links remaining for inode %r: 0; removing.', self.name)
self.storage._remove_inode(self.name)
self.tree.remove()
self.storage._autocommit()
class GitStorage(object):
commit_author = "Spaghetti User <*****@*****.**>"
@classmethod
def create(cls, repo_path):
if not os.path.isdir(repo_path):
os.mkdir(repo_path)
eg = EasyGit.new_repo(repo_path, bare=True)
inodes = eg.root.new_tree('inodes')
root_ls = eg.root.new_blob('root.ls')
root_sub = eg.root.new_tree('root.sub')
features_blob = eg.root.new_blob('features')
features_blob.data = '{}'
features = FeatureBlob(features_blob)
features['next_inode_number'] = 1
features['inode_index_format'] = 'treetree'
features['inode_format'] = 'treetree'
eg.commit(cls.commit_author, 'Created empty filesystem')
return cls(repo_path)
def __init__(self, repo_path, autocommit=True):
self.eg = EasyGit.open_repo(repo_path)
features = FeatureBlob(self.eg.root['features'])
assert features.get('inode_format', None) == 'treetree'
assert features.get('inode_index_format', None) == 'treetree'
self.autocommit = autocommit
log.debug('Loaded storage, autocommit=%r, HEAD=%r',
autocommit, self.eg.get_head_id())
self._inode_cache = {}
self._inodes_tt = TreeTree(self.eg.root['inodes'], prefix='it')
def get_root(self):
commit_tree = self.eg.root
root_ls = commit_tree['root.ls']
root_sub = commit_tree['root.sub']
root = StorageDir('root', root_ls, root_sub, '/', self, None)
root.path = '/'
return root
def get_inode(self, name):
if name in self._inode_cache:
inode = self._inode_cache[name]()
if inode is None:
del self._inode_cache[name]
else:
return inode
inode_tree = self._inodes_tt[name[1:]]
inode = StorageInode(name, inode_tree, self)
self._inode_cache[name] = weakref.ref(inode)
return inode
def create_inode(self):
features = FeatureBlob(self.eg.root['features'])
next_inode_number = features['next_inode_number']
features['next_inode_number'] = next_inode_number + 1
inode_name = 'i%d' % next_inode_number
inode_tree = self._inodes_tt.new_tree(inode_name[1:])
inode_tree.new_blob('meta').data = StorageInode.default_meta
return self.get_inode(inode_name)
def _remove_inode(self, name):
if name in self._inode_cache:
del self._inode_cache[name]
def _autocommit(self):
if self.autocommit:
self.commit("Auto commit")
def commit(self, message=None, amend=False, head_id=None, branch='master'):
log.info('Committing')
if head_id is None:
head_id = self.eg.get_head_id(branch)
if amend:
git = self.eg.git
prev_commit = git.commit(head_id)
parents = prev_commit.parents
if message is None:
message = prev_commit.message
else:
parents = [head_id]
assert message is not None
self.eg.commit(self.commit_author, message, parents,
branch=branch)
def __init__(self, name, tree, storage):
self.name = name
self.tree = tree
self.storage = storage
self.tt = TreeTree(tree, prefix='bt')
log.debug('Loaded inode %r', name)
class StorageInode(object):
blocksize = 64*1024 # 64 KB
#blocksize = 1024*1024 # 1024 KB
default_meta = ('mode: 0100644\n'
'nlink: 1\n'
'uid: 0\n'
'gid: 0\n'
'size: 0\n')
int_meta = ('nlink', 'uid', 'gid', 'size')
oct_meta = ('mode',)
def __init__(self, name, tree, storage):
self.name = name
self.tree = tree
self.storage = storage
self.tt = TreeTree(tree, prefix='bt')
log.debug('Loaded inode %r', name)
def _read_meta(self):
try:
meta_blob = self.tree['meta']
except KeyError:
meta_raw = self.default_meta
else:
meta_raw = meta_blob.data
return dict(line.split(': ', 1)
for line in meta_raw.strip().split('\n'))
def _write_meta(self, meta_data):
meta_raw = ''.join('%s: %s\n' % (key, value)
for key, value in sorted(meta_data.items()))
self.tree.new_blob('meta').data = meta_raw
self.storage._autocommit()
def __getitem__(self, key):
value = self._read_meta()[key]
if key in self.oct_meta:
value = int(value, base=8)
elif key in self.int_meta:
value = int(value)
return value
def __setitem__(self, key, value):
if key in self.oct_meta:
value = '0%o' % value
elif key in self.int_meta:
value = '%d' % value
else:
raise NotImplementedError
meta_data = self._read_meta()
meta_data[key] = value
self._write_meta(meta_data)
def read_block(self, n):
block_name = str(n)
log.debug('Reading block %r of inode %r', block_name, self.name)
try:
block = self.tt[block_name]
except KeyError:
return ''
else:
return block.data
def write_block(self, n, data):
block_name = str(n)
log.debug('Writing block %r of inode %r', block_name, self.name)
try:
block = self.tt[block_name]
except KeyError:
block = self.tt.new_blob(block_name)
block.data = data
self.storage._autocommit()
def delete_block(self, n):
block_name = str(n)
log.debug('Removing block %r of inode %r', block_name, self.name)
del self.tt[block_name]
self.storage._autocommit()
def read_data(self, offset, length):
end = offset + length
eof = self['size']
if end > eof:
end = eof
length = end - offset
if length <= 0:
return ''
first_block = offset / self.blocksize
last_block = end / self.blocksize
output = StringIO()
for n_block in range(first_block, last_block+1):
block_offset = n_block * self.blocksize
fragment_offset = 0
if n_block == first_block:
fragment_offset = offset - block_offset
fragment_end = self.blocksize
if n_block == last_block:
fragment_end = end - block_offset
block_data = self.read_block(n_block)
fragment = block_data[fragment_offset:fragment_end]
assert len(fragment) == fragment_end - fragment_offset
output.write(fragment)
output = output.getvalue()
assert len(output) == length
return output
def write_data(self, data, offset):
current_size = self['size']
if current_size < offset:
self.truncate(offset)
log.info('Inode %s writing %d bytes at offset %d',
repr(self.name), len(data), offset)
end = offset + len(data)
first_block = offset / self.blocksize
last_block = end / self.blocksize
for n_block in range(first_block, last_block+1):
block_offset = n_block * self.blocksize
insert_offset = 0
if n_block == first_block:
insert_offset = offset - block_offset
insert_end = self.blocksize
if n_block == last_block:
insert_end = end - block_offset
data_start = block_offset + insert_offset - offset
data_end = block_offset + insert_end - offset
log.debug('Updating inode %d between (%d, %d) '
'with data slice between (%d, %d)',
n_block, insert_offset, insert_end,
data_start, data_end)
current_data = self.read_block(n_block)
datafile = StringIO()
datafile.write(current_data)
datafile.seek(insert_offset)
datafile.write(data[data_start:data_end])
self.write_block(n_block, datafile.getvalue())
if end > current_size:
self['size'] = end
def truncate(self, new_size):
log.info("Truncating inode %s, new size %d", repr(self.name), new_size)
current_size = self['size']
if current_size < new_size:
# TODO: avoid creating one big string
self.write_data('\0' * (new_size - current_size), current_size)
elif current_size > new_size:
first_block = new_size / self.blocksize
last_block = current_size / self.blocksize
truncate_offset = new_size % self.blocksize
for n_block in range(first_block, last_block+1):
if n_block == first_block and truncate_offset > 0:
old_data = self.read_block(n_block)
self.write_block(n_block, old_data[:truncate_offset])
else:
self.delete_block(n_block)
self['size'] = new_size
def unlink(self):
log.info('Unlinking inode %r', self.name)
nlink = self['nlink'] - 1
if nlink > 0:
log.info('Links remaining for inode %r: %d', self.name, nlink)
self['nlink'] = nlink
else:
log.info('Links remaining for inode %r: 0; removing.', self.name)
self.storage._remove_inode(self.name)
self.tree.remove()
self.storage._autocommit()
