def __init__(self, db_name="d", host = None, port = None):
self.model_has_been_created = False
try:
if not host and not port:
self.storage = ZODB.FileStorage.FileStorage(file_name=db_name, blob_dir='.blob')
self.db = ZODB.DB(self.storage)
else:
self.storage = ClientStorage.ClientStorage((host,port))
self.db = ZODB.DB(self.storage)
self.connection = self.db.open()
self.root = self.connection.root()
_init(self)
self.users = self.root['users']
self.piadas = self.root['piadas']
self.admin = self.root['admin']
pass
except Exception as e:
raise Exception("Database Error - " + str(e))
def db(self, db_name):
"""
Construct ZODB database if not existing, store DB object
:param db_name: name of new db, including path
"""
is_existing_db = isfile(db_name)
if is_existing_db:
print("Opening db {db_name}".format(db_name=db_name))
storage = ZODB.FileStorage.FileStorage(db_name, read_only=True)
self._db = ZODB.DB(storage)
with self._db.transaction() as conn:
self.width = conn.root.width
self.nb_pos = conn.root.nb_pos
self.nb_neg = conn.root.nb_neg
#~ print(conn.root.neg[630309][0], conn.root.neg[630309][1])
#~ print(len(conn.root.neg[360354][0]), len(conn.root.neg[360354][1]))
print("Width: ", self.width, "\t# pos: ", self.nb_pos, "\t# neg: ", self.nb_neg)
else:
storage = ZODB.FileStorage.FileStorage(db_name, read_only=False)
self._db = ZODB.DB(storage)
with self._db.transaction() as conn:
conn.root.width = self.width
conn.root.nb_pos = 0
conn.root.nb_neg = 0
conn.root.pos = BTrees.IOBTree.BTree() # IOB is for integer - is this about number or measurements - yes!
conn.root.neg = BTrees.IOBTree.BTree()
def db(s):
"""ZODB connection pool.
"""
import atexit
zuri = s.config.get('JOB_DB')
if zuri is None:
z = ZODB.DB("simsvc.fs")
else:
sf, kws = zodburi.resolve_uri(zuri)
z = ZODB.DB(sf(), **kws)
atexit.register(z.close)
return z
def run(storage):
if hasattr(storage, 'is_connected'):
while not storage.is_connected():
time.sleep(CONNECT_DELAY)
pid = os.getpid()
print "Client process connected:", pid, storage
db = ZODB.DB(storage)
root = db.open().root()
while 1:
try:
s = root[pid] = Stats()
transaction.commit()
except ZODB.POSException.ConflictError:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
break
dict = root["multi"]
prev = None
i = 0
while i < RECORDS_PER_CLIENT:
try:
size = len(dict)
r = dict[size] = Record(pid, size)
if prev:
prev.set_next(r)
transaction.commit()
except ZODB.POSException.ConflictError, err:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
i = i + 1
if VERBOSE and (i < 5 or i % 10 == 0):
print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
def test_prefetch(self):
db = ZODB.DB(None)
fetched = []
def prefetch(oids, tid):
fetched.append((list(map(u64, oids)), tid))
db.storage.prefetch = prefetch
with db.transaction() as conn:
for i in range(10):
conn.root()[i] = conn.root().__class__()
conn = db.open()
conn.prefetch(z64)
conn.prefetch([z64])
conn.prefetch(conn.root())
conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3])
self.assertEqual(fetched,
[([0], conn._storage._start),
([0], conn._storage._start),
([0], conn._storage._start),
([0, 1, 2, 3, 4], conn._storage._start),
])
db.close()
def on_post(self, req, resp, token):
trello = TrelloApi(trelloconfig.api_key, token=trelloconfig.token)
trello_token = trello.get_token_url('My App',
expires='30days',
write_access=True)
storage = ZODB.FileStorage.FileStorage('trees/' + token + '.fs')
db = ZODB.DB(storage)
connection = db.open()
root = connection.root
if hasattr(root, 'tree'):
tree = root.tree
else:
resp.body = "Initialize first"
connection.close()
db.close()
storage.close()
return
lst = list(btree.inorder(tree))
connection.close()
db.close()
storage.close()
if len(lst) > 0:
id_new = trello.boards.new_list('Le5vKw7H', token)['id']
for card in lst:
trello.cards.new(card, id_new)
def getPeople(self, options):
folder = tempfile.gettempdir()
#folder = './' # my /tmp is a tmpfs
fname = os.path.join(folder, 'performance_data.fs')
if options.reload:
try:
os.remove(fname)
except:
pass
fs = ZODB.FileStorage.FileStorage(fname)
db = ZODB.DB(fs)
conn = db.open()
root = conn.root()
if options.reload:
root['people'] = people = PeopleZ()
transaction.commit()
# Profile inserts
transaction.begin()
t1 = time.time()
for idx in xrange(options.size):
klass = (self.personKlass if (MULTIPLE_CLASSES and idx % 2)
else self.person2Klass)
name = 'Mr Number %.5i' % idx
people[name] = klass(name, random.randint(0, 100))
transaction.commit()
t2 = time.time()
self.printResult('Insert', t1, t2, options.size)
else:
people = root['people']
return people
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
def setUp(self):
self.__path = os.path.abspath('test.fs')
store = ZODB.FileStorage.FileStorage(self.__path)
self.db = ZODB.DB(store)
warnings.filterwarnings('ignore',
message='Versions are deprecated',
module=__name__)
def _get_connection(cls):
try:
return cls.Meta.connection
except AttributeError:
zodb = ZODB.DB(None)
cls.Meta.connection = zodb.open()
return cls.Meta.connection
def start(self):
self.started = 1
self.path = tempfile.mktemp(suffix=".fs")
self._storage = FileStorage(self.path)
self.db = ZODB.DB(self._storage)
self.do_updates()
self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
def sandbox(base=None):
'''Returns a sandbox copy of the base ZODB.'''
if base is None: base = Zope2.DB
base_storage = base._storage
quota = getattr(base_storage, '_quota', None)
storage = DemoStorage(base=base_storage, quota=quota)
return ZODB.DB(storage)
def get_zodb_root():
global __zodb_root
if not __zodb_root:
db = ZODB.DB('data/db.fs')
connection = db.open()
__zodb_root = connection.root
return __zodb_root
def get_stored_structure_data():
try:
f = FfAA.Settings.file_for_storing_structures
d = os.path.split(f)[0]
if not os.path.exists(d):
print "Creating directory", d
os.mkdir(d)
storage = ZODB.FileStorage.FileStorage(f)
db = ZODB.DB(storage)
conn = db.open()
dbroot = conn.root()
if not dbroot.has_key('structures'):
print_color(3, "Add structures to database\n")
dbroot['structures'] = Structures()
structures = dbroot['structures']
init = Initializer(structures)
except ZODB.POSException.StorageSystemError:
print "Sound Structure database already opened by another process."
db = structures = init = None
return db, structures, init
def setUp(self):
# set up a zodb
# we can't use DemoStorage here 'cos it doesn't support History
self.dir = tempfile.mkdtemp()
self.s = FileStorage(os.path.join(self.dir, 'testHistory.fs'),
create=True)
self.connection = ZODB.DB(self.s).open()
r = self.connection.root()
a = Application()
r['Application'] = a
self.root = a
# create a python script
manage_addPythonScript(a, 'test')
self.ps = ps = a.test
# commit some changes
ps.write('return 1')
t = transaction.get()
# undo note made by Application instantiation above.
t.description = None
t.note('Change 1')
t.commit()
ps.write('return 2')
t = transaction.get()
t.note('Change 2')
t.commit()
ps.write('return 3')
t = transaction.get()
t.note('Change 3')
t.commit()
def testExport(self):
# create a neo storage
self.neo.start()
(neo_db, neo_conn) = self.neo.getZODBConnection()
self.__populate(neo_db)
dump = self.__dump(neo_db.storage)
# copy neo to data fs
dfs_storage = self.__getDataFS(reset=True)
neo_storage = self.neo.getZODBStorage()
dfs_storage.copyTransactionsFrom(neo_storage)
# check data fs content
dfs_db = ZODB.DB(dfs_storage)
root = dfs_db.open().root()
self.__checkTree(root['trees'])
dfs_db.close()
self.neo.stop()
self.neo = NEOCluster(db_list=['test_neo1'],
partitions=3,
importer=[("root", {
"storage":
"<filestorage>\npath %s\n</filestorage>" %
dfs_storage.getName()
})],
temp_dir=self.getTempDirectory())
self.neo.start()
neo_db, neo_conn = self.neo.getZODBConnection()
self.__checkTree(neo_conn.root()['trees'])
self.assertEqual(dump, self.__dump(neo_db.storage))
def open_database(self):
"""Create the internal engine and session builder
for this manager based on the configured database"""
if self.sessions and self.db is not None:
raise RuntimeError(
"cannot change database after sessions are established")
# Connect/open the database
factory_class, factory_args = zodburi.resolve_uri(self.config["db"])
storage = factory_class()
self.db = ZODB.DB(storage, **factory_args)
conn = self.db.open()
if not hasattr(conn.root, "targets"):
conn.root.targets = persistent.list.PersistentList()
if not hasattr(conn.root, "history"):
conn.root.history = persistent.list.PersistentList()
conn.transaction_manager.commit()
conn.close()
# Rebuild the command parser now that the database is available
self.parser = CommandParser(self)
def setUp(self):
# Our default transaction manager is
# transaction._manager.ThreadTransactionManager
# so no need to set it.
ZODB.tests.util.TestCase.setUp(self)
self._storage = ZODB.FileStorage.FileStorage('ZODBTests.fs', create=1)
self._db = ZODB.DB(self._storage)
class RN_Manager:
storage = FS.FileStorage(
os.path.join('SamurAI', 'database', 'IAs', 'index.fs'))
db = ZODB.DB(storage)
conn = db.open()
dbroot = conn.root()
ultimo_da_fila = dbroot['ultimo'] #if 'ultimo' in dbroot else 0
ias = dbroot['ias']
def __init__(self):
pass
@classmethod
def proximo_indice(cls):
if len(cls.ias) == MAX_RNs:
return cls.ultimo_da_fila
return len(cls.ias)
@classmethod
def registrar(cls, nome_arquivo):
if len(cls.ias) == MAX_RNs:
# registra o novo arquivo
cls.ias[cls.ultimo_da_fila] = nome_arquivo
# avança o ponteiro para o proximo da fila
cls.ultimo_da_fila = (cls.ultimo_da_fila + 1) % MAX_RNs
cls.dbroot['ultimo'] = cls.ultimo_da_fila
else:
cls.ias[len(cls.ias)] = nome_arquivo
transaction.commit()
@classmethod
def close(cls):
cls.conn.close()
cls.db.close()
cls.storage.close()
def get_N_urls(main_url, N):
try:
storage = ZODB.FileStorage.FileStorage('data.fs')
db = ZODB.DB(storage)
connection = db.open()
root = connection.root
except:
# Невозможно получить данные, предварительно не скачав их
db.close()
print("Cannot 'get' before 'load'")
return
if main_url in root.urls:
l = list(root.urls[main_url][1])
else:
# Когда запрашиваемой ссылки нет в базе. Это значит, что ссылка
# принадлежит к третьему уровню вложенности (не обрабатывалась)
# или такой ссылки вообще не было в загруженных данных
l = []
db.close()
print("No such URL in the DB")
return
if N > len(l):
N = len(l)
for i in range(N):
url = l[i]
try:
title = root.urls[url][0]
except:
# Ссылка третьего уровня - для неё нет данных кроме самого URL
title = "Not found in DB"
print(url + ': "' + title + '"')
db.close()
def __init__(self, storage_path):
self.storage = ZODB.FileStorage.FileStorage(storage_path)
self.database = ZODB.DB(self.storage)
self.connection = self.database.open()
self.root = self.connection.root
try:
self.root.scryfall_cards
except (AttributeError, KeyError):
self.root.scryfall_cards = PCardList()
try:
self.root.scryfall_sets
except (AttributeError, KeyError):
self.root.scryfall_sets = PSetList()
try:
self.root.mtgio_cards
except (AttributeError, KeyError):
self.root.mtgio_cards = PCardList()
try:
self.root.mtgio_sets
except (AttributeError, KeyError):
self.root.mtgio_sets = PSetList()
def __init__(self, file="data/work/Data.fs", open=1):
self.db = ZODB.DB(ZODB.FileStorage.FileStorage(file))
if open == 1:
self.connection = self.db.open()
self.root = self.connection.root()
def main():
storage = ZODB.FileStorage.FileStorage("pupper.db")
db = ZODB.DB(storage)
with db.transaction() as c:
for host in ("scc", "root", "puppettest5.scc.net.davepedu.com"):
if "foo2" in c.root.nodes[host].classes:
del c.root.nodes[host].classes["foo2"]
def setUp(self):
# set up a zodb
# we can't use DemoStorage here 'cos it doesn't support History
self.dir = tempfile.mkdtemp()
fs_path = os.path.join(self.dir, 'testHistory.fs')
self.s = FileStorage(fs_path, create=True)
self.connection = ZODB.DB(self.s).open()
r = self.connection.root()
a = Application()
r['Application'] = a
self.root = a
# create a python script
a['test'] = HistoryItem()
self.hi = hi = a.test
# commit some changes
hi.title = 'First title'
t = transaction.get()
# undo note made by Application instantiation above.
t.description = None
t.note(u'Change 1')
t.commit()
time.sleep(0.02) # wait at least one Windows clock tick
hi.title = 'Second title'
t = transaction.get()
t.note(u'Change 2')
t.commit()
time.sleep(0.02) # wait at least one Windows clock tick
hi.title = 'Third title'
t = transaction.get()
t.note(u'Change 3')
t.commit()
def open(self):
import ZODB
from ZODB.FileStorage import FileStorage
self.path = self.conf['rdf.store_conf']
openstr = os.path.abspath(self.path)
try:
fs = FileStorage(openstr)
except IOError:
L.exception("Failed to create a FileStorage")
raise ZODBSourceOpenFailError(openstr)
self.zdb = ZODB.DB(fs, cache_size=1600)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph('ZODB')
self.graph = root['rdflib']
try:
transaction.commit()
except Exception:
# catch commit exception and close db.
# otherwise db would stay open and follow up tests
# will detect the db in error state
L.exception('Forced to abort transaction on ZODB store opening',
exc_info=True)
transaction.abort()
transaction.begin()
self.graph.open(self.path)
def __init__(self, filename, realm=None):
"""
:param str filename: Config file with users and their password hashes
:param str realm: ZODB's default permissions realm
"""
self.storage_filename = path.splitext(filename)[0] + ".db"
self.storage = FileStorage.FileStorage(self.storage_filename)
self.db = ZODB.DB(self.storage)
self.db_conn = self.db.open()
self.db_root = self.db_conn.root()
root = self.db_root
with transaction.manager:
if not "users" in root:
root["users"] = IdStore() # uid -> user
if not "usernames" in root:
root["usernames"] = self.family.OI.BTree() # username -> uid
self.filename = filename
self.load()
# Frankly speaking, this realm-based security is questionable
# Keep it here for now
if realm:
if self.realm and self.realm != realm:
raise ValueError("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
self.noncekey = rand(32)
def open(self, databases=None):
section = self.config
storage = section.storage.open()
options = {}
def _option(name, oname=None):
v = getattr(section, name)
if v is not None:
if oname is None:
oname = name
options[oname] = v
_option('pool_timeout')
_option('allow_implicit_cross_references', 'xrefs')
_option('large_record_size')
try:
return ZODB.DB(
storage,
pool_size=section.pool_size,
cache_size=section.cache_size,
cache_size_bytes=section.cache_size_bytes,
historical_pool_size=section.historical_pool_size,
historical_cache_size=section.historical_cache_size,
historical_cache_size_bytes=section.historical_cache_size_bytes,
historical_timeout=section.historical_timeout,
database_name=section.database_name or self.name or '',
databases=databases,
**options)
except:
storage.close()
raise
def testMigrationTool(self):
dfs_storage = self.__getDataFS()
dfs_db = ZODB.DB(dfs_storage)
self.__populate(dfs_db, with_undo=False)
dump = self.__dump(dfs_storage)
fs_path = dfs_storage.__name__
dfs_db.close()
neo = self.neo
neo.start()
kw = {'cluster': neo.cluster_name, 'quiet': None}
master_nodes = neo.master_nodes.replace('/', ' ')
if neo.SSL:
kw['ca'], kw['cert'], kw['key'] = neo.SSL
p = NEOProcess('neomigrate', fs_path, master_nodes, **kw)
p.start()
p.wait()
os.remove(fs_path)
p = NEOProcess('neomigrate', master_nodes, fs_path, **kw)
p.start()
p.wait()
self.assertEqual(dump, self.__dump(FileStorage(fs_path)))
def setUp(self):
if self.url.endswith('.fs'):
from ZODB.FileStorage import FileStorage
if os.path.exists(self.path):
os.unlink('/tmp/zodb_local3.fs')
os.unlink('/tmp/zodb_local3.fs.index')
os.unlink('/tmp/zodb_local3.fs.tmp')
os.unlink('/tmp/zodb_local3.fs.lock')
openstr = os.path.abspath(os.path.expanduser(self.url[7:]))
fs = FileStorage(openstr)
else:
from ZEO.ClientStorage import ClientStorage
schema, opts = _parse_rfc1738_args(self.url)
fs = ClientStorage((opts['host'], int(opts['port'])))
self.zdb = ZODB.DB(fs)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph(self.store_name)
self.graph = self.g = root['rdflib']
self.michel = URIRef('michel')
self.tarek = URIRef('tarek')
self.bob = URIRef('bob')
self.likes = URIRef('likes')
self.hates = URIRef('hates')
self.pizza = URIRef('pizza')
self.cheese = URIRef('cheese')
transaction.commit()
def open(self):
import ZODB
from ZODB.FileStorage import FileStorage
from zc.lockfile import LockError
self.path = self.conf['rdf.store_conf']
openstr = os.path.abspath(self.path)
try:
fs = FileStorage(openstr)
except IOError:
L.exception("Failed to create a FileStorage")
raise ZODBSourceOpenFailError(openstr)
except LockError:
L.exception('Found database "{}" is locked when trying to open it. '
'The PID of this process: {}'.format(openstr, os.getpid()), exc_info=True)
raise DatabaseConflict('Database ' + openstr + ' locked')
self.zdb = ZODB.DB(fs, cache_size=1600)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph('ZODB')
self.graph = root['rdflib']
try:
transaction.commit()
except Exception:
# catch commit exception and close db.
# otherwise db would stay open and follow up tests
# will detect the db in error state
L.exception('Forced to abort transaction on ZODB store opening', exc_info=True)
transaction.abort()
transaction.begin()
self.graph.open(self.path)
