def __init__(self, db_name="d", host = None, port = None):
self.model_has_been_created = False
try:
if not host and not port:
self.storage = ZODB.FileStorage.FileStorage(file_name=db_name, blob_dir='.blob')
self.db = ZODB.DB(self.storage)
else:
self.storage = ClientStorage.ClientStorage((host,port))
self.db = ZODB.DB(self.storage)
self.connection = self.db.open()
self.root = self.connection.root()
_init(self)
self.users = self.root['users']
self.piadas = self.root['piadas']
self.admin = self.root['admin']
pass
except Exception as e:
raise Exception("Database Error - " + str(e))
def db(self, db_name):
"""
Construct ZODB database if not existing, store DB object
:param db_name: name of new db, including path
"""
is_existing_db = isfile(db_name)
if is_existing_db:
print("Opening db {db_name}".format(db_name=db_name))
storage = ZODB.FileStorage.FileStorage(db_name, read_only=True)
self._db = ZODB.DB(storage)
with self._db.transaction() as conn:
self.width = conn.root.width
self.nb_pos = conn.root.nb_pos
self.nb_neg = conn.root.nb_neg
#~ print(conn.root.neg[630309][0], conn.root.neg[630309][1])
#~ print(len(conn.root.neg[360354][0]), len(conn.root.neg[360354][1]))
print("Width: ", self.width, "\t# pos: ", self.nb_pos, "\t# neg: ", self.nb_neg)
else:
storage = ZODB.FileStorage.FileStorage(db_name, read_only=False)
self._db = ZODB.DB(storage)
with self._db.transaction() as conn:
conn.root.width = self.width
conn.root.nb_pos = 0
conn.root.nb_neg = 0
conn.root.pos = BTrees.IOBTree.BTree() # IOB is for integer - is this about number or measurements - yes!
conn.root.neg = BTrees.IOBTree.BTree()
def db(s):
"""ZODB connection pool.
"""
import atexit
zuri = s.config.get('JOB_DB')
if zuri is None:
z = ZODB.DB("simsvc.fs")
else:
sf, kws = zodburi.resolve_uri(zuri)
z = ZODB.DB(sf(), **kws)
atexit.register(z.close)
return z
def test_find_node_correlations_A():
"""
`find_node_correlations`
Feature A: correctly terminating if insufficient attractors found.
"""
# Test for {no attractors found, one attractor found, two attractors found}.
aggregated_attractors_1 = dict()
aggregated_attractors_2 = {
1: construct_aggregated_attractor(attractor_states_1, 10, 1, 0)
}
aggregated_attractors_3 = {
1: construct_aggregated_attractor(attractor_states_1, 1, 1, 0),
8: construct_aggregated_attractor(attractor_states_2, 1, 1, 0)
}
for aggregated_attractors in [
aggregated_attractors_1, aggregated_attractors_2,
aggregated_attractors_3
]:
db_conn = ZODB.connection(None)
init_attractor_db_structure(db_conn)
for key, aggregated_attractor in aggregated_attractors.items():
db_conn.root.aggregated_attractors[key] = aggregated_attractor
db_conn.root.n_aggregated_attractors.change(1)
db_conn.root.total_frequency.change(aggregated_attractor.frequency)
node_correlations = find_node_correlations(db_conn, None, True)
test_description = generate_test_description(locals(),
'aggregated_attractors')
assert node_correlations is None, test_description
def get_stored_structure_data():
try:
f = FfAA.Settings.file_for_storing_structures
d = os.path.split(f)[0]
if not os.path.exists(d):
print "Creating directory", d
os.mkdir(d)
storage = ZODB.FileStorage.FileStorage(f)
db = ZODB.DB(storage)
conn = db.open()
dbroot = conn.root()
if not dbroot.has_key('structures'):
print_color(3, "Add structures to database\n")
dbroot['structures'] = Structures()
structures = dbroot['structures']
init = Initializer(structures)
except ZODB.POSException.StorageSystemError:
print "Sound Structure database already opened by another process."
db = structures = init = None
return db, structures, init
def setUp(self):
# set up a zodb
# we can't use DemoStorage here 'cos it doesn't support History
self.dir = tempfile.mkdtemp()
self.s = FileStorage(os.path.join(self.dir, 'testHistory.fs'),
create=True)
self.connection = ZODB.DB(self.s).open()
r = self.connection.root()
a = Application()
r['Application'] = a
self.root = a
# create a python script
manage_addPythonScript(a, 'test')
self.ps = ps = a.test
# commit some changes
ps.write('return 1')
t = transaction.get()
# undo note made by Application instantiation above.
t.description = None
t.note('Change 1')
t.commit()
ps.write('return 2')
t = transaction.get()
t.note('Change 2')
t.commit()
ps.write('return 3')
t = transaction.get()
t.note('Change 3')
t.commit()
def testExport(self):
# create a neo storage
self.neo.start()
(neo_db, neo_conn) = self.neo.getZODBConnection()
self.__populate(neo_db)
dump = self.__dump(neo_db.storage)
# copy neo to data fs
dfs_storage = self.__getDataFS(reset=True)
neo_storage = self.neo.getZODBStorage()
dfs_storage.copyTransactionsFrom(neo_storage)
# check data fs content
dfs_db = ZODB.DB(dfs_storage)
root = dfs_db.open().root()
self.__checkTree(root['trees'])
dfs_db.close()
self.neo.stop()
self.neo = NEOCluster(db_list=['test_neo1'],
partitions=3,
importer=[("root", {
"storage":
"<filestorage>\npath %s\n</filestorage>" %
dfs_storage.getName()
})],
temp_dir=self.getTempDirectory())
self.neo.start()
neo_db, neo_conn = self.neo.getZODBConnection()
self.__checkTree(neo_conn.root()['trees'])
self.assertEqual(dump, self.__dump(neo_db.storage))
def _get_connection(cls):
try:
return cls.Meta.connection
except AttributeError:
zodb = ZODB.DB(None)
cls.Meta.connection = zodb.open()
return cls.Meta.connection
def get_zodb_root():
global __zodb_root
if not __zodb_root:
db = ZODB.DB('data/db.fs')
connection = db.open()
__zodb_root = connection.root
return __zodb_root
def setUp(self):
if self.url.endswith('.fs'):
from ZODB.FileStorage import FileStorage
if os.path.exists(self.path):
os.unlink('/tmp/zodb_local3.fs')
os.unlink('/tmp/zodb_local3.fs.index')
os.unlink('/tmp/zodb_local3.fs.tmp')
os.unlink('/tmp/zodb_local3.fs.lock')
openstr = os.path.abspath(os.path.expanduser(self.url[7:]))
fs = FileStorage(openstr)
else:
from ZEO.ClientStorage import ClientStorage
schema, opts = _parse_rfc1738_args(self.url)
fs = ClientStorage((opts['host'], int(opts['port'])))
self.zdb = ZODB.DB(fs)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph(self.store_name)
self.graph = self.g = root['rdflib']
self.michel = URIRef('michel')
self.tarek = URIRef('tarek')
self.bob = URIRef('bob')
self.likes = URIRef('likes')
self.hates = URIRef('hates')
self.pizza = URIRef('pizza')
self.cheese = URIRef('cheese')
transaction.commit()
def open_database(self):
"""Create the internal engine and session builder
for this manager based on the configured database"""
if self.sessions and self.db is not None:
raise RuntimeError(
"cannot change database after sessions are established")
# Connect/open the database
factory_class, factory_args = zodburi.resolve_uri(self.config["db"])
storage = factory_class()
self.db = ZODB.DB(storage, **factory_args)
conn = self.db.open()
if not hasattr(conn.root, "targets"):
conn.root.targets = persistent.list.PersistentList()
if not hasattr(conn.root, "history"):
conn.root.history = persistent.list.PersistentList()
conn.transaction_manager.commit()
conn.close()
# Rebuild the command parser now that the database is available
self.parser = CommandParser(self)
def open(self):
import ZODB
from ZODB.FileStorage import FileStorage
from zc.lockfile import LockError
self.path = self.conf['rdf.store_conf']
openstr = os.path.abspath(self.path)
try:
fs = FileStorage(openstr)
except IOError:
L.exception("Failed to create a FileStorage")
raise ZODBSourceOpenFailError(openstr)
except LockError:
L.exception('Found database "{}" is locked when trying to open it. '
'The PID of this process: {}'.format(openstr, os.getpid()), exc_info=True)
raise DatabaseConflict('Database ' + openstr + ' locked')
self.zdb = ZODB.DB(fs, cache_size=1600)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph('ZODB')
self.graph = root['rdflib']
try:
transaction.commit()
except Exception:
# catch commit exception and close db.
# otherwise db would stay open and follow up tests
# will detect the db in error state
L.exception('Forced to abort transaction on ZODB store opening', exc_info=True)
transaction.abort()
transaction.begin()
self.graph.open(self.path)
def testMigrationTool(self):
dfs_storage = self.__getDataFS()
dfs_db = ZODB.DB(dfs_storage)
self.__populate(dfs_db, with_undo=False)
dump = self.__dump(dfs_storage)
fs_path = dfs_storage.__name__
dfs_db.close()
neo = self.neo
neo.start()
kw = {'cluster': neo.cluster_name, 'quiet': None}
master_nodes = neo.master_nodes.replace('/', ' ')
if neo.SSL:
kw['ca'], kw['cert'], kw['key'] = neo.SSL
p = NEOProcess('neomigrate', fs_path, master_nodes, **kw)
p.start()
p.wait()
os.remove(fs_path)
p = NEOProcess('neomigrate', master_nodes, fs_path, **kw)
p.start()
p.wait()
self.assertEqual(dump, self.__dump(FileStorage(fs_path)))
def __init__(self, filename, realm=None):
"""
:param str filename: Config file with users and their password hashes
:param str realm: ZODB's default permissions realm
"""
self.storage_filename = path.splitext(filename)[0] + ".db"
self.storage = FileStorage.FileStorage(self.storage_filename)
self.db = ZODB.DB(self.storage)
self.db_conn = self.db.open()
self.db_root = self.db_conn.root()
root = self.db_root
with transaction.manager:
if not "users" in root:
root["users"] = IdStore() # uid -> user
if not "usernames" in root:
root["usernames"] = self.family.OI.BTree() # username -> uid
self.filename = filename
self.load()
# Frankly speaking, this realm-based security is questionable
# Keep it here for now
if realm:
if self.realm and self.realm != realm:
raise ValueError("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
self.noncekey = rand(32)
def setUp(self):
# set up a zodb
# we can't use DemoStorage here 'cos it doesn't support History
self.dir = tempfile.mkdtemp()
fs_path = os.path.join(self.dir, 'testHistory.fs')
self.s = FileStorage(fs_path, create=True)
self.connection = ZODB.DB(self.s).open()
r = self.connection.root()
a = Application()
r['Application'] = a
self.root = a
# create a python script
a['test'] = HistoryItem()
self.hi = hi = a.test
# commit some changes
hi.title = 'First title'
t = transaction.get()
# undo note made by Application instantiation above.
t.description = None
t.note(u'Change 1')
t.commit()
time.sleep(0.02) # wait at least one Windows clock tick
hi.title = 'Second title'
t = transaction.get()
t.note(u'Change 2')
t.commit()
time.sleep(0.02) # wait at least one Windows clock tick
hi.title = 'Third title'
t = transaction.get()
t.note(u'Change 3')
t.commit()
def __init__(self, file="data/work/Data.fs", open=1):
self.db = ZODB.DB(ZODB.FileStorage.FileStorage(file))
if open == 1:
self.connection = self.db.open()
self.root = self.connection.root()
def get_root():
"""
Return the root object of the current database.
"""
db_path = click.get_app_dir('katalog', force_posix=True)
connection = ZODB.connection(db_path + '/data.fs')
return connection.root()
def sandbox(base=None):
'''Returns a sandbox copy of the base ZODB.'''
if base is None: base = Zope2.DB
base_storage = base._storage
quota = getattr(base_storage, '_quota', None)
storage = DemoStorage(base=base_storage, quota=quota)
return ZODB.DB(storage)
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
def open(self):
import ZODB
from ZODB.FileStorage import FileStorage
self.path = self.conf['rdf.store_conf']
openstr = os.path.abspath(self.path)
try:
fs = FileStorage(openstr)
except IOError:
L.exception("Failed to create a FileStorage")
raise ZODBSourceOpenFailError(openstr)
self.zdb = ZODB.DB(fs, cache_size=1600)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph('ZODB')
self.graph = root['rdflib']
try:
transaction.commit()
except Exception:
# catch commit exception and close db.
# otherwise db would stay open and follow up tests
# will detect the db in error state
L.exception('Forced to abort transaction on ZODB store opening',
exc_info=True)
transaction.abort()
transaction.begin()
self.graph.open(self.path)
def main():
storage = ZODB.FileStorage.FileStorage("pupper.db")
db = ZODB.DB(storage)
with db.transaction() as c:
for host in ("scc", "root", "puppettest5.scc.net.davepedu.com"):
if "foo2" in c.root.nodes[host].classes:
del c.root.nodes[host].classes["foo2"]
def setUp(self):
self.__path = os.path.abspath('test.fs')
store = ZODB.FileStorage.FileStorage(self.__path)
self.db = ZODB.DB(store)
warnings.filterwarnings('ignore',
message='Versions are deprecated',
module=__name__)
def __init__(self, storage_path):
self.storage = ZODB.FileStorage.FileStorage(storage_path)
self.database = ZODB.DB(self.storage)
self.connection = self.database.open()
self.root = self.connection.root
try:
self.root.scryfall_cards
except (AttributeError, KeyError):
self.root.scryfall_cards = PCardList()
try:
self.root.scryfall_sets
except (AttributeError, KeyError):
self.root.scryfall_sets = PSetList()
try:
self.root.mtgio_cards
except (AttributeError, KeyError):
self.root.mtgio_cards = PCardList()
try:
self.root.mtgio_sets
except (AttributeError, KeyError):
self.root.mtgio_sets = PSetList()
def setUp(self):
# Our default transaction manager is
# transaction._manager.ThreadTransactionManager
# so no need to set it.
ZODB.tests.util.TestCase.setUp(self)
self._storage = ZODB.FileStorage.FileStorage('ZODBTests.fs', create=1)
self._db = ZODB.DB(self._storage)
def getPeople(self, options):
folder = tempfile.gettempdir()
#folder = './' # my /tmp is a tmpfs
fname = os.path.join(folder, 'performance_data.fs')
if options.reload:
try:
os.remove(fname)
except:
pass
fs = ZODB.FileStorage.FileStorage(fname)
db = ZODB.DB(fs)
conn = db.open()
root = conn.root()
if options.reload:
root['people'] = people = PeopleZ()
transaction.commit()
# Profile inserts
transaction.begin()
t1 = time.time()
for idx in xrange(options.size):
klass = (self.personKlass if (MULTIPLE_CLASSES and idx % 2)
else self.person2Klass)
name = 'Mr Number %.5i' % idx
people[name] = klass(name, random.randint(0, 100))
transaction.commit()
t2 = time.time()
self.printResult('Insert', t1, t2, options.size)
else:
people = root['people']
return people
def start(self):
self.started = 1
self.path = tempfile.mktemp(suffix=".fs")
self._storage = FileStorage(self.path)
self.db = ZODB.DB(self._storage)
self.do_updates()
self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
def setUp(self):
from ..site import Site
self.site = Site("Test")
self.generation = self.site.generation
self.conn = ZODB.connection(None)
self.conn.root.site = self.site
self.conn.transaction_manager.commit()
class RN_Manager:
storage = FS.FileStorage(
os.path.join('SamurAI', 'database', 'IAs', 'index.fs'))
db = ZODB.DB(storage)
conn = db.open()
dbroot = conn.root()
ultimo_da_fila = dbroot['ultimo'] #if 'ultimo' in dbroot else 0
ias = dbroot['ias']
def __init__(self):
pass
@classmethod
def proximo_indice(cls):
if len(cls.ias) == MAX_RNs:
return cls.ultimo_da_fila
return len(cls.ias)
@classmethod
def registrar(cls, nome_arquivo):
if len(cls.ias) == MAX_RNs:
# registra o novo arquivo
cls.ias[cls.ultimo_da_fila] = nome_arquivo
# avança o ponteiro para o proximo da fila
cls.ultimo_da_fila = (cls.ultimo_da_fila + 1) % MAX_RNs
cls.dbroot['ultimo'] = cls.ultimo_da_fila
else:
cls.ias[len(cls.ias)] = nome_arquivo
transaction.commit()
@classmethod
def close(cls):
cls.conn.close()
cls.db.close()
cls.storage.close()
def open(self, databases=None):
section = self.config
storage = section.storage.open()
options = {}
def _option(name, oname=None):
v = getattr(section, name)
if v is not None:
if oname is None:
oname = name
options[oname] = v
_option('pool_timeout')
_option('allow_implicit_cross_references', 'xrefs')
_option('large_record_size')
try:
return ZODB.DB(
storage,
pool_size=section.pool_size,
cache_size=section.cache_size,
cache_size_bytes=section.cache_size_bytes,
historical_pool_size=section.historical_pool_size,
historical_cache_size=section.historical_cache_size,
historical_cache_size_bytes=section.historical_cache_size_bytes,
historical_timeout=section.historical_timeout,
database_name=section.database_name or self.name or '',
databases=databases,
**options)
except:
storage.close()
raise
def run(storage):
if hasattr(storage, 'is_connected'):
while not storage.is_connected():
time.sleep(CONNECT_DELAY)
pid = os.getpid()
print "Client process connected:", pid, storage
db = ZODB.DB(storage)
root = db.open().root()
while 1:
try:
s = root[pid] = Stats()
transaction.commit()
except ZODB.POSException.ConflictError:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
break
dict = root["multi"]
prev = None
i = 0
while i < RECORDS_PER_CLIENT:
try:
size = len(dict)
r = dict[size] = Record(pid, size)
if prev:
prev.set_next(r)
transaction.commit()
except ZODB.POSException.ConflictError, err:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
i = i + 1
if VERBOSE and (i < 5 or i % 10 == 0):
print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
def test_prefetch(self):
db = ZODB.DB(None)
fetched = []
def prefetch(oids, tid):
fetched.append((list(map(u64, oids)), tid))
db.storage.prefetch = prefetch
with db.transaction() as conn:
for i in range(10):
conn.root()[i] = conn.root().__class__()
conn = db.open()
conn.prefetch(z64)
conn.prefetch([z64])
conn.prefetch(conn.root())
conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3])
self.assertEqual(fetched,
[([0], conn._storage._start),
([0], conn._storage._start),
([0], conn._storage._start),
([0, 1, 2, 3, 4], conn._storage._start),
])
db.close()
def test_prefetch_optional_imvcc(self):
conn = ZODB.connection(MVCCMappingStorage())
conn.prefetch(z64)
conn.prefetch([z64])
conn.prefetch(conn.root())
conn.prefetch(z64, [z64])
conn.prefetch(z64, [z64], conn.root())
conn.close()
def test_prefetch_optional(self):
conn = ZODB.connection(None)
conn.prefetch(z64)
conn.prefetch([z64])
conn.prefetch(conn.root())
conn.prefetch(z64, [z64])
conn.prefetch(z64, [z64], conn.root())
conn.close()
def opendb(self, contName=None):
""" Open the database if not yet,
return the required container.
"""
if self.conn is None:
self.conn = ZODB.connection(self.db_path)
if contName is None:
contName = self.contName
if contName is None:
raise "must specify a container name"
return self.getContainer(self.conn.root, contName)
def opendb(self):
""" self.db is a dictionary-like object, the initial db
will be empty, thus this statement: not self.db,
will always yield a True, which cause the process try
to open the DB again even if it did open it before, and
cause an exception (locking problem).
The correct way is compare it directly with None object.
"""
if self.db is None:
connection = ZODB.connection(self.db_path)
self.db = getContainer(connection.root, 'main')
def test_used_by_connection(self):
import ZODB
from ZODB.MappingStorage import MappingStorage
class Storage(MappingStorage):
def tpc_begin(self, transaction):
self.test_transaction = transaction
return MappingStorage.tpc_begin(self, transaction)
storage = Storage()
conn = ZODB.connection(storage)
with conn.transaction_manager as t:
t.user = u'user\x80'
t.description = u'description\x80'
t.setExtendedInfo('foo', 'FOO')
conn.root.x = 1
t = storage.test_transaction
self.assertEqual(t.__class__, TransactionMetaData)
self.assertEqual(t.user, b'user\xc2\x80')
self.assertEqual(t.description, b'description\xc2\x80')
self.assertEqual(t.extension, dict(foo='FOO'))
def _open_db(path):
conn = _connections.get(path)
if not (conn and _db_is_opened(conn)):
conn = ZODB.connection(path)
_connections[path] = conn
return conn
