def _scenario(self):
c1 = Connection(self._get_storage())
c2 = Connection(self._get_storage())
c1.get_root()['A'] = Persistent()
c1.get_root()['B'] = Persistent()
c1.get_root()['A'].a = 1
c1.commit()
c2.abort()
c1.cache.recent_objects.discard(c1.get_root()['A'])
# Imagine c1 has been running for a while, and
# cache management, for example, has caused the
# cache reference to be weak.
return c1, c2
def interactive_client(file, address, cache_size, readonly, repair,
startup, storage_class=None):
if file:
storage = get_storage(file, storage_class=storage_class,
readonly=readonly, repair=repair)
description = file
else:
socket_address = SocketAddress.new(address)
wait_for_server(address=socket_address)
storage = ClientStorage(address=socket_address)
description = socket_address
connection = Connection(storage, cache_size=cache_size)
console_module = ModuleType('__console__')
sys.modules['__console__'] = console_module
namespace = {'connection': connection,
'root': connection.get_root(),
'get': connection.get,
'sys': sys,
'os': os,
'int8_to_str': int8_to_str,
'str_to_int8': str_to_int8,
'pp': pprint}
vars(console_module).update(namespace)
configure_readline(
vars(console_module), os.path.expanduser("~/.durushistory"))
console = InteractiveConsole(vars(console_module))
if startup:
console.runsource('execfile("%s")' % os.path.expanduser(startup))
help = (' connection -> the Connection\n'
' root -> the root instance')
console.interact('Durus %s\n%s' % (description, help))
def delete(self):
connection = Connection(MemoryStorage())
pd = PersistentDict((x, True) for x in range(10))
connection.root['x'] = pd
connection.commit()
del pd[1]
assert pd._p_is_unsaved()
def copy(self):
connection = Connection(MemoryStorage())
pd = PersistentDict((x, True) for x in range(10))
pd2 = pd.copy()
assert pd == pd2
pd[1] = 34
assert pd != pd2
def check_storage_tools(self):
connection = Connection(self._get_storage())
root = connection.get_root()
root['a'] = Persistent()
root['b'] = Persistent()
connection.commit()
index = get_reference_index(connection.get_storage())
assert index == {
int8_to_str(1): [int8_to_str(0)],
int8_to_str(2): [int8_to_str(0)]
}
census = get_census(connection.get_storage())
assert census == {
as_bytes('PersistentDict'): 1,
as_bytes('Persistent'): 2
}
references = list(
gen_referring_oid_record(connection.get_storage(), int8_to_str(1)))
assert references == [(int8_to_str(0),
connection.get_storage().load(int8_to_str(0)))]
class Fake(object):
pass
s = Fake()
s.__class__ = Storage
raises(RuntimeError, s.__init__)
raises(NotImplementedError, s.load, None)
raises(NotImplementedError, s.begin)
raises(NotImplementedError, s.store, None, None)
raises(NotImplementedError, s.end)
raises(NotImplementedError, s.sync)
g = s.gen_oid_record()
raises(NotImplementedError, next, g)
def run_trials():
numTrials = 3000
gens = 1000
from multiprocessing.pool import ThreadPool as Pool
pool = Pool(50)
jids = pool.map(f, [gens] * numTrials)
print "Done spawning trials. Retrieving results..."
results = pool.map(cloud_result, jids)
firstLocusFreqsHists = zeros((numTrials, gens + 1), dtype='float')
lastLocusFreqsHists = zeros((numTrials, gens + 1), dtype='float')
print "Done retrieving results. Press Enter to serialize..."
raw_input()
for i, result in enumerate(results):
firstLocusFreqsHists[i, :], lastLocusFreqsHists[i, :] = result
with closing(FileStorage("soda_results.durus")) as durus:
conn = Connection(durus)
conn.get_root()[str(int(floor(time.time())))] = (firstLocusFreqsHists,
lastLocusFreqsHists)
conn.commit()
pool.close()
pool.join()
def check_connection(self):
self.conn = conn = Connection(self._get_storage())
self.root = root = conn.get_root()
assert root._p_is_ghost() == False
assert root is conn.get(int8_to_str(0))
assert root is conn.get(0)
assert conn is root._p_connection
assert conn.get(int8_to_str(1)) == None
conn.abort()
conn.commit()
assert root._p_is_ghost() == False
root['a'] = Persistent()
assert root._p_is_unsaved() == True
assert root['a']._p_is_unsaved() == True
root['a'].f = 2
assert list(conn.changed.values()) == [root]
conn.commit()
assert root._p_is_saved()
assert list(conn.changed.values()) == []
root['a'] = Persistent()
assert list(conn.changed.values()) == [root]
root['b'] = Persistent()
root['a'].a = 'a'
root['b'].b = 'b'
conn.commit()
root['a'].a = 'a'
root['b'].b = 'b'
conn.abort()
conn.shrink_cache()
root['b'].b = 'b'
del conn
def render_results(timestamp=None):
with closing(FileStorage("soda_results.durus")) as durus:
conn = Connection(durus)
db = conn.get_root()
if not timestamp:
timestamp = sorted(db.keys())[-1]
firstLocusFreqsHists, lastLocusFreqsHists = db[timestamp]
print "Done deserializing results. Plotting..."
x = [(2, 'First', firstLocusFreqsHists, "effective"),
(3, 'Last', lastLocusFreqsHists, "non-effective")]
for i, pos, freqsHists, filename in x:
freqsHists = freqsHists[:, :801]
f = figure(i)
hold(False)
plot(transpose(freqsHists), color='grey')
hold(True)
maxGens = freqsHists.shape[1] - 1
plot([0, maxGens], [.05, .05], 'k--')
plot([0, maxGens], [.95, .95], 'k--')
axis([0, maxGens, 0, 1])
xlabel('Generation')
ylabel('1-Frequency of the ' + pos + ' Locus')
f.canvas.draw()
f.show()
savefig(filename + '.png', format='png', dpi=200)
def pack_storage_main():
parser = OptionParser()
parser.set_description("Packs a Durus storage.")
parser.add_option(
'--file',
dest="file",
default=None,
help="If this is not given, the storage is through a Durus server.")
parser.add_option('--port',
dest="port",
default=DEFAULT_PORT,
type="int",
help="Port the server is on. (default=%s)" %
DEFAULT_PORT)
parser.add_option('--host',
dest="host",
default=DEFAULT_HOST,
help="Host of the server. (default=%s)" % DEFAULT_HOST)
(options, args) = parser.parse_args()
if options.file is None:
wait_for_server(options.host, options.port)
storage = ClientStorage(host=options.host, port=options.port)
else:
storage = FileStorage(options.file)
connection = Connection(storage)
connection.pack()
def __init__(self, file, new):
self.__file = file
if new:
if os.path.exists(self.__file):
os.remove(self.__file)
self.__connection = Connection(FileStorage(self.__file))
self.__root = self.__connection.get_root()
def __init__(self):
self._conn = Connection(FileStorage(PROJECTS_DATA_PATH))
self._data = self._conn.get_root()
if not len(self._data.keys()):
self._data["Default"] = PersistentDict(
autocomplete=PersistentDict())
self.sync()
def check_conflict(self):
b = Connection(self._get_storage())
c = Connection(self._get_storage())
rootb = b.get(int8_to_str(0))
rootb['b'] = Persistent()
rootc = c.get(int8_to_str(0))
rootc['c'] = Persistent()
c.commit()
raises(ConflictError, b.commit)
raises(KeyError, rootb.__getitem__, 'c')
transaction_serial = b.transaction_serial
b.abort()
assert b.get_transaction_serial() > transaction_serial
assert rootb._p_is_ghost()
rootc['d'] = Persistent()
c.commit()
rootb['d']
def get_config():
connection = Connection(FileStorage("/var/tmp/test.durus"))
root = connection.get_root() # connection set as shown above.
if not root.has_key("_pconfig"):
cf = GenericConfiguration()
root["_pconfig"] = cf
root["_pconfig"]["default"] = SECTION()
connection.commit()
return root["_pconfig"]
def main(old_file, new_file):
if old_file.startswith('-'):
usage()
if new_file.startswith('-'):
usage()
assert not exists(new_file)
connection = Connection(sys.argv[1])
tmpfile = TemporaryFile()
print("pickling from " + old_file)
dump(connection.get_root().__getstate__(), tmpfile, 2)
connection = None
tmpfile.seek(0)
connection2 = Connection(sys.argv[2])
print("unpickling")
connection2.get_root().__setstate__(load(tmpfile))
connection2.get_root()._p_note_change()
print("commit to " + new_file)
connection2.commit()
print("pack")
connection2.pack()
def __init__(self):
# durus file storage
self.conndurus = Connection(FileStorage(CONFIG['durus_file']))
root = self.conndurus.get_root()
if not root.get('users'):
root['users'] = PersistentDict() # {user jid: CUser}
if not root.get('feeds'):
root['feeds'] = CFeeds()
self.data = root['users']
self.feeds = root['feeds']
self.save()
def main():
parser = OptionParser()
parser.set_description('Stress test a Durus Server')
parser.add_option('--port',
dest='port',
default=DEFAULT_PORT,
type='int',
help='Port to listen on. (default=%s)' % DEFAULT_PORT)
parser.add_option('--host',
dest='host',
default=DEFAULT_HOST,
help='Host to listen on. (default=%s)' % DEFAULT_HOST)
parser.add_option('--cache_size',
dest="cache_size",
default=4000,
type="int",
help="Size of client cache (default=4000)")
parser.add_option('--max-loops',
dest='loops',
default=None,
type='int',
help='Maximum number of loops before exiting.')
(options, args) = parser.parse_args()
from durus.logger import logger
logger.setLevel(5)
storage = ClientStorage(host=options.host, port=options.port)
connection = Connection(storage, cache_size=options.cache_size)
try:
if 'obj' not in connection.get_root():
init_db(connection)
verify_db(connection, all=True)
connection.commit()
except ConflictError:
connection.abort()
n = options.loops
while n is None or n > 0:
if n is not None:
n -= 1
try:
if hasattr(sys, 'gettotalrefcount'):
sys.stdout.write('refs = %s\n' % sys.gettotalrefcount())
if randbool():
connection.abort()
verify_db(connection)
mutate_db(connection)
connection.commit()
maybe_sleep()
except ConflictError:
sys.stdout.write('conflict\n')
connection.abort()
maybe_sleep()
def check_touch_every_reference(self):
connection = Connection(self._get_storage())
root = connection.get_root()
root['a'] = Persistent()
root['b'] = Persistent()
from durus.persistent_list import PersistentList
root['b'].c = PersistentList()
connection.commit()
touch_every_reference(connection, 'PersistentList')
assert root['b']._p_is_unsaved()
assert root['b'].c._p_is_unsaved()
assert not root._p_is_unsaved()
assert len(list(connection.get_cache())) == 4
def create_durus_publisher():
global connection
filename = os.path.join(tempfile.gettempdir(), 'quixote-demo.durus')
print('Opening %r as a Durus database.' % filename)
connection = Connection(FileStorage(filename))
root = connection.get_root()
session_manager = root.get('session_manager', None)
if session_manager is None:
session_manager = PersistentSessionManager()
connection.get_root()['session_manager'] = session_manager
connection.commit()
return Publisher(RootDirectory(),
session_manager=session_manager,
display_exceptions='plain')
def check_fine_conflict(self):
c1 = Connection(self._get_storage())
c2 = Connection(self._get_storage())
c1.get_root()['A'] = Persistent()
c1.get_root()['A'].a = 1
c1.get_root()['B'] = Persistent()
c1.commit()
c2.abort()
# c1 has A loaded.
assert not c1.get_root()['A']._p_is_ghost()
c1.get_root()['B'].b = 1
c2.get_root()['A'].a = 2
c2.commit()
# Even though A has been changed by c2,
# c1 has not accessed an attribute of A since
# the last c1.commit(), so we don't want a ConflictError.
c1.commit()
assert c1.get_root()['A']._p_is_ghost()
c1.get_root()['A'].a # accessed!
c1.get_root()['B'].b = 1
c2.get_root()['A'].a = 2
c2.commit()
raises(WriteConflictError, c1.commit)
def lowlevelops(self):
from durus.persistent import _getattribute, _setattribute
from durus.persistent import _delattribute, _hasattribute
storage = TempFileStorage()
connection = Connection(storage)
root = connection.get_root()
root._p_set_status_ghost()
assert not _hasattribute(root, 'data')
root._p_set_status_ghost()
raises(AttributeError, _getattribute, root, 'data')
assert root._p_is_ghost()
_setattribute(root, 'data', 'bogus')
assert root._p_is_ghost()
_delattribute(root, 'data')
assert root._p_is_ghost()
def check_more(self):
storage = TempFileStorage()
connection = Connection(storage)
root=connection.get_root()
assert not root._p_is_ghost()
root['a'] = 1
assert root._p_is_unsaved()
del root['a']
connection.abort()
assert root._p_is_ghost()
raises(AttributeError, getattr, root, 'a')
root._p_set_status_saved()
assert root._p_is_saved()
root._p_set_status_unsaved()
assert root._p_is_unsaved()
root._p_set_status_ghost()
assert root._p_is_ghost()
root._p_set_status_unsaved()
def b(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
new_oid = s.new_oid()
assert new_oid == deleted_oid
new_oid = s.new_oid()
assert new_oid == int8_to_str(11)
def check_shrink(self):
storage = self._get_storage()
self.conn = conn = Connection(storage, cache_size=3)
self.root = root = conn.get_root()
root['a'] = Persistent()
root['b'] = Persistent()
root['c'] = Persistent()
assert self.root._p_is_unsaved()
conn.commit()
root['a'].a = 1
conn.commit()
root['b'].b = 1
root['c'].c = 1
root['d'] = Persistent()
root['e'] = Persistent()
root['f'] = Persistent()
conn.commit()
root['f'].f = 1
root['g'] = Persistent()
conn.commit()
conn.pack()
def main():
from shutil import copyfile
from os.path import exists
def usage():
sys.stdout.write("Usage: python %s <existing_file> <new_file>\n" %
sys.argv[0])
sys.stdout.write(" Creates a new py3k-compatible file ")
sys.stdout.write("from an existing FileStorage file.\n")
raise SystemExit
if len(sys.argv) != 3:
usage()
infile = sys.argv[1]
outfile = sys.argv[2]
if not exists(infile):
usage()
if exists(outfile):
if input('overwrite %r? [y/N] ' % outfile).strip().lower() != 'y':
raise SystemExit
copyfile(infile, outfile)
# monkey patch pickler class, must be done before importing durus stuff
patch_pickler()
from durus.__main__ import get_storage_class
from durus.connection import Connection
storage_class = get_storage_class(outfile)
storage = storage_class(outfile)
connection = Connection(storage)
print("Converting %s for use with py3k." % outfile)
for j, x in enumerate(connection.get_crawler()):
x._p_note_change()
if j > 0 and j % 10000 == 0:
print(j)
connection.commit()
print(j)
connection.commit()
connection.pack()
def a(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oids = [
r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid
]
del r['a0']
del r['a2']
del r['a7']
del r['a8']
c.commit()
c.pack()
c.abort()
assert c.get(deleted_oids[0])._p_is_ghost()
assert c.get(deleted_oids[1])._p_is_ghost()
raises(KeyError, getattr, c.get(deleted_oids[0]), 'a')
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
c.commit()
c.pack()
new_oid = s.new_oid()
assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == int8_to_str(11), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(12), repr(new_oid)
def c(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
r.clear()
c.commit()
c.pack()
c.abort()
new_oid = s.new_oid()
assert new_oid == int8_to_str(1), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(2), repr(new_oid)
def showExperimentTimeStamps():
with closing(FileStorage("soda_results.durus")) as durus:
conn = Connection(durus)
return conn.get_root().keys()
import sys
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
stream=file('log.txt', 'w'))
def construct_mud(objstorethunk):
"""Construct a MUD factory."""
return ConnectionFactory(objstorethunk)
def run_mud(mud, port):
"""Run the MUD factory."""
reactor.listenTCP(port, mud)
mud.ticker.add_command(commit_gameworld)
mud_ticker.add_command(event_flusher)
mud.ticker.start()
logging.info("OK, setup done, handing you over to the reactor's loop!")
sys.stdout.write("Server is up and running.")
reactor.run()
if __name__ == '__main__':
#this needs to be wrapped in a lambda, because Durus is quite eager to load
#stuff. If it wasn't so eager, the ConnectionFactory would just pull what
#it needed when, rather than getting silly errors on the next line.
connection = lambda: Connection(FileStorage("mudlib.durus"))
run_mud(construct_mud(connection), 6666)
def __init__(self, filename):
self._connection = Connection(FileStorage(filename))
def __init__(self, host, port):
self._connection = Connection(ClientStorage(host=host, port=port))
