def make_keyfs():
keyfs = KeyFS(tmpdir, storage)
key1 = keyfs.add_key("NAME1", "hello", int)
keyfs.notifier.on_key_change(key1, queue.put)
pool = ThreadPool()
pool.register(keyfs.notifier)
pool.start()
try:
yield key1
finally:
pool.shutdown()
def test_keyfs_sqlite(gentmp):
from devpi_server import keyfs_sqlite
tmp = gentmp()
keyfs = KeyFS(tmp, keyfs_sqlite.Storage)
with keyfs.transaction(write=True) as tx:
assert tx.conn.io_file_os_path('foo') is None
tx.conn.io_file_set('foo', b'bar')
tx.conn._sqlconn.commit()
with keyfs.transaction(write=False) as tx:
assert tx.conn.io_file_os_path('foo') is None
assert tx.conn.io_file_get('foo') == b'bar'
assert [x.basename for x in tmp.listdir()] == ['.sqlite_db']
def keyfs(self):
from devpi_server.keyfs import KeyFS
from devpi_server.model import add_keys
keyfs = KeyFS(self.config.serverdir,
self.config.storage,
readonly=self.is_replica(),
cache_size=self.config.args.keyfs_cache_size)
add_keys(self, keyfs)
keyfs.finalize_init()
if not self.config.args.requests_only:
self.thread_pool.register(keyfs.notifier)
return keyfs
def test_keyfs_sqlite_fs(gentmp):
from devpi_server import keyfs_sqlite_fs
tmp = gentmp()
keyfs = KeyFS(tmp, keyfs_sqlite_fs.Storage)
with keyfs.transaction(write=True) as tx:
assert tx.conn.io_file_os_path('foo') == tmp.join('foo').strpath
tx.conn.io_file_set('foo', b'bar')
tx.conn._sqlconn.commit()
with keyfs.transaction(write=False) as tx:
assert tx.conn.io_file_get('foo') == b'bar'
with open(tx.conn.io_file_os_path('foo'), 'rb') as f:
assert f.read() == b'bar'
assert sorted(x.basename for x in tmp.listdir()) == ['.sqlite', 'foo']
def keyfs(self):
from devpi_server.keyfs import KeyFS
from devpi_server.model import add_keys
keyfs = KeyFS(self.config.serverdir,
self.config.storage,
readonly=self.is_replica(),
cache_size=self.config.args.keyfs_cache_size)
add_keys(self, keyfs)
try:
keyfs.finalize_init()
except Exception:
threadlog.exception("Error while trying to initialize storage")
fatal("Couldn't initialize storage")
if not self.config.requests_only:
self.thread_pool.register(keyfs.notifier)
return keyfs
def keyfs(self):
from devpi_server.keyfs import KeyFS
from devpi_server.model import add_keys
keyfs = KeyFS(self.config.serverdir, readonly=self.is_replica())
add_keys(self, keyfs)
self.thread_pool.register(keyfs.notifier)
return keyfs
def test_keyfs_readonly(self, storage, tmpdir):
keyfs = KeyFS(tmpdir, storage, readonly=True)
with pytest.raises(keyfs.ReadOnly):
with keyfs.transaction(write=True):
pass
assert not hasattr(keyfs, "tx")
with pytest.raises(keyfs.ReadOnly):
with keyfs.transaction():
keyfs.restart_as_write_transaction()
with pytest.raises(keyfs.ReadOnly):
keyfs.begin_transaction_in_thread(write=True)
with keyfs.transaction():
pass
def test_import_changes_subscriber_error(self, keyfs, storage, tmpdir):
pkey = keyfs.add_key("NAME", "hello/{name}", dict)
D = pkey(name="world")
with keyfs.transaction(write=True):
D.set({1:1})
new_keyfs = KeyFS(tmpdir.join("newkeyfs"), storage)
pkey = new_keyfs.add_key("NAME", "hello/{name}", dict)
new_keyfs.subscribe_on_import(pkey, lambda *args: 0/0)
serial = new_keyfs.get_current_serial()
with keyfs.transaction() as tx:
changes = tx.conn.get_changes(0)
with pytest.raises(ZeroDivisionError):
new_keyfs.import_changes(0, changes)
assert new_keyfs.get_current_serial() == serial
def test_import_changes_subscriber(self, keyfs, storage, tmpdir):
pkey = keyfs.add_key("NAME", "hello/{name}", dict)
D = pkey(name="world")
with keyfs.transaction(write=True):
D.set({1:1})
assert keyfs.get_current_serial() == 0
# load entries into new keyfs instance
new_keyfs = KeyFS(tmpdir.join("newkeyfs"), storage)
pkey = new_keyfs.add_key("NAME", "hello/{name}", dict)
l = []
new_keyfs.subscribe_on_import(pkey, lambda *args: l.append(args))
with keyfs.transaction() as tx:
changes = tx.conn.get_changes(0)
new_keyfs.import_changes(0, changes)
assert l[0][1:] == (new_keyfs.NAME(name="world"), {1:1}, -1)
def test_import_changes(self, keyfs, storage, tmpdir):
D = keyfs.add_key("NAME", "hello", dict)
with keyfs.transaction(write=True):
D.set({1:1})
with keyfs.transaction(write=True):
D.delete()
with keyfs.transaction(write=True):
D.set({2:2})
with keyfs._storage.get_connection() as conn:
serial = conn.last_changelog_serial
assert serial == 2
# load entries into new keyfs instance
new_keyfs = KeyFS(tmpdir.join("newkeyfs"), storage)
D2 = new_keyfs.add_key("NAME", "hello", dict)
for serial in range(3):
with keyfs.transaction() as tx:
changes = tx.conn.get_changes(serial)
new_keyfs.import_changes(serial, changes)
with new_keyfs.transaction() as tx:
assert tx.get_value_at(D2, 0) == {1:1}
with pytest.raises(KeyError):
assert tx.get_value_at(D2, 1)
assert tx.get_value_at(D2, 2) == {2:2}
def keyfs(gentmp, pool, storage):
keyfs = KeyFS(gentmp(), storage)
pool.register(keyfs.notifier)
yield keyfs
def keyfs(self):
from devpi_server.keyfs import KeyFS
keyfs = KeyFS(self.config.serverdir)
add_keys(keyfs)
return keyfs
tw.line("%s: %s" % (key, revlist))
if __name__ == "__main__":
tw = py.io.TerminalWriter()
parser = argparse.ArgumentParser()
parser.add_argument("--serverdir",
type=str,
default="~/.devpi/server",
help="serverdir directory to look at")
parser.add_argument("--gc",
action="store_true",
help="garbage collect analysis")
parser.add_argument("changelog",
type=int,
help="changelog to look at",
nargs="*")
args = parser.parse_args()
basedir = py.path.local(args.serverdir, expanduser=True)
assert basedir.exists()
keyfs = KeyFS(basedir)
tw.line("keyfs at %s" % basedir, bold=True)
if args.gc:
garbage_collect_analysis(keyfs)
elif args.changelog:
for serial in args.changelog:
tw.sep("-", "serial %s" % serial)
pprint.pprint(keyfs._fs.get_changes(int(serial)))
else:
tw.line("number of changelogs: %s" % keyfs.get_next_serial())
import py
from devpi_server.keyfs import KeyFS
def create_dict(keyfs, num):
key = keyfs.add_key("NAME", "somedir/file", dict)
for i in range(num):
with keyfs.transaction():
key.set({i: i})
def create_files(keyfs, num):
key = keyfs.add_key("FILES", "somedir/otherfile", bytes)
content = '1'.encode('utf-8') * 5000000
for i in range(num):
with keyfs.transaction():
key.set(content)
print "wrote", i
if __name__ == "__main__":
dir = py.path.local("/tmp/keyfs")
if dir.exists():
dir.remove()
keyfs = KeyFS(dir)
#create_dict(keyfs, 3000)
create_files(keyfs, 150)
def keyfs(gentmp):
return KeyFS(gentmp())