def __init__(self, db_name="d", host = None, port = None): self.model_has_been_created = False try: if not host and not port: self.storage = ZODB.FileStorage.FileStorage(file_name=db_name, blob_dir='.blob') self.db = ZODB.DB(self.storage) else: self.storage = ClientStorage.ClientStorage((host,port)) self.db = ZODB.DB(self.storage) self.connection = self.db.open() self.root = self.connection.root() _init(self) self.users = self.root['users'] self.piadas = self.root['piadas'] self.admin = self.root['admin'] pass except Exception as e: raise Exception("Database Error - " + str(e))
def db(self, db_name): """ Construct ZODB database if not existing, store DB object :param db_name: name of new db, including path """ is_existing_db = isfile(db_name) if is_existing_db: print("Opening db {db_name}".format(db_name=db_name)) storage = ZODB.FileStorage.FileStorage(db_name, read_only=True) self._db = ZODB.DB(storage) with self._db.transaction() as conn: self.width = conn.root.width self.nb_pos = conn.root.nb_pos self.nb_neg = conn.root.nb_neg #~ print(conn.root.neg[630309][0], conn.root.neg[630309][1]) #~ print(len(conn.root.neg[360354][0]), len(conn.root.neg[360354][1])) print("Width: ", self.width, "\t# pos: ", self.nb_pos, "\t# neg: ", self.nb_neg) else: storage = ZODB.FileStorage.FileStorage(db_name, read_only=False) self._db = ZODB.DB(storage) with self._db.transaction() as conn: conn.root.width = self.width conn.root.nb_pos = 0 conn.root.nb_neg = 0 conn.root.pos = BTrees.IOBTree.BTree() # IOB is for integer - is this about number or measurements - yes! conn.root.neg = BTrees.IOBTree.BTree()
def db(s): """ZODB connection pool. """ import atexit zuri = s.config.get('JOB_DB') if zuri is None: z = ZODB.DB("simsvc.fs") else: sf, kws = zodburi.resolve_uri(zuri) z = ZODB.DB(sf(), **kws) atexit.register(z.close) return z
def test_find_node_correlations_A(): """ `find_node_correlations` Feature A: correctly terminating if insufficient attractors found. """ # Test for {no attractors found, one attractor found, two attractors found}. aggregated_attractors_1 = dict() aggregated_attractors_2 = { 1: construct_aggregated_attractor(attractor_states_1, 10, 1, 0) } aggregated_attractors_3 = { 1: construct_aggregated_attractor(attractor_states_1, 1, 1, 0), 8: construct_aggregated_attractor(attractor_states_2, 1, 1, 0) } for aggregated_attractors in [ aggregated_attractors_1, aggregated_attractors_2, aggregated_attractors_3 ]: db_conn = ZODB.connection(None) init_attractor_db_structure(db_conn) for key, aggregated_attractor in aggregated_attractors.items(): db_conn.root.aggregated_attractors[key] = aggregated_attractor db_conn.root.n_aggregated_attractors.change(1) db_conn.root.total_frequency.change(aggregated_attractor.frequency) node_correlations = find_node_correlations(db_conn, None, True) test_description = generate_test_description(locals(), 'aggregated_attractors') assert node_correlations is None, test_description
def get_stored_structure_data(): try: f = FfAA.Settings.file_for_storing_structures d = os.path.split(f)[0] if not os.path.exists(d): print "Creating directory", d os.mkdir(d) storage = ZODB.FileStorage.FileStorage(f) db = ZODB.DB(storage) conn = db.open() dbroot = conn.root() if not dbroot.has_key('structures'): print_color(3, "Add structures to database\n") dbroot['structures'] = Structures() structures = dbroot['structures'] init = Initializer(structures) except ZODB.POSException.StorageSystemError: print "Sound Structure database already opened by another process." db = structures = init = None return db, structures, init
def setUp(self): # set up a zodb # we can't use DemoStorage here 'cos it doesn't support History self.dir = tempfile.mkdtemp() self.s = FileStorage(os.path.join(self.dir, 'testHistory.fs'), create=True) self.connection = ZODB.DB(self.s).open() r = self.connection.root() a = Application() r['Application'] = a self.root = a # create a python script manage_addPythonScript(a, 'test') self.ps = ps = a.test # commit some changes ps.write('return 1') t = transaction.get() # undo note made by Application instantiation above. t.description = None t.note('Change 1') t.commit() ps.write('return 2') t = transaction.get() t.note('Change 2') t.commit() ps.write('return 3') t = transaction.get() t.note('Change 3') t.commit()
def testExport(self): # create a neo storage self.neo.start() (neo_db, neo_conn) = self.neo.getZODBConnection() self.__populate(neo_db) dump = self.__dump(neo_db.storage) # copy neo to data fs dfs_storage = self.__getDataFS(reset=True) neo_storage = self.neo.getZODBStorage() dfs_storage.copyTransactionsFrom(neo_storage) # check data fs content dfs_db = ZODB.DB(dfs_storage) root = dfs_db.open().root() self.__checkTree(root['trees']) dfs_db.close() self.neo.stop() self.neo = NEOCluster(db_list=['test_neo1'], partitions=3, importer=[("root", { "storage": "<filestorage>\npath %s\n</filestorage>" % dfs_storage.getName() })], temp_dir=self.getTempDirectory()) self.neo.start() neo_db, neo_conn = self.neo.getZODBConnection() self.__checkTree(neo_conn.root()['trees']) self.assertEqual(dump, self.__dump(neo_db.storage))
def _get_connection(cls): try: return cls.Meta.connection except AttributeError: zodb = ZODB.DB(None) cls.Meta.connection = zodb.open() return cls.Meta.connection
def get_zodb_root(): global __zodb_root if not __zodb_root: db = ZODB.DB('data/db.fs') connection = db.open() __zodb_root = connection.root return __zodb_root
def setUp(self): if self.url.endswith('.fs'): from ZODB.FileStorage import FileStorage if os.path.exists(self.path): os.unlink('/tmp/zodb_local3.fs') os.unlink('/tmp/zodb_local3.fs.index') os.unlink('/tmp/zodb_local3.fs.tmp') os.unlink('/tmp/zodb_local3.fs.lock') openstr = os.path.abspath(os.path.expanduser(self.url[7:])) fs = FileStorage(openstr) else: from ZEO.ClientStorage import ClientStorage schema, opts = _parse_rfc1738_args(self.url) fs = ClientStorage((opts['host'], int(opts['port']))) self.zdb = ZODB.DB(fs) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph(self.store_name) self.graph = self.g = root['rdflib'] self.michel = URIRef('michel') self.tarek = URIRef('tarek') self.bob = URIRef('bob') self.likes = URIRef('likes') self.hates = URIRef('hates') self.pizza = URIRef('pizza') self.cheese = URIRef('cheese') transaction.commit()
def open_database(self): """Create the internal engine and session builder for this manager based on the configured database""" if self.sessions and self.db is not None: raise RuntimeError( "cannot change database after sessions are established") # Connect/open the database factory_class, factory_args = zodburi.resolve_uri(self.config["db"]) storage = factory_class() self.db = ZODB.DB(storage, **factory_args) conn = self.db.open() if not hasattr(conn.root, "targets"): conn.root.targets = persistent.list.PersistentList() if not hasattr(conn.root, "history"): conn.root.history = persistent.list.PersistentList() conn.transaction_manager.commit() conn.close() # Rebuild the command parser now that the database is available self.parser = CommandParser(self)
def open(self): import ZODB from ZODB.FileStorage import FileStorage from zc.lockfile import LockError self.path = self.conf['rdf.store_conf'] openstr = os.path.abspath(self.path) try: fs = FileStorage(openstr) except IOError: L.exception("Failed to create a FileStorage") raise ZODBSourceOpenFailError(openstr) except LockError: L.exception('Found database "{}" is locked when trying to open it. ' 'The PID of this process: {}'.format(openstr, os.getpid()), exc_info=True) raise DatabaseConflict('Database ' + openstr + ' locked') self.zdb = ZODB.DB(fs, cache_size=1600) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph('ZODB') self.graph = root['rdflib'] try: transaction.commit() except Exception: # catch commit exception and close db. # otherwise db would stay open and follow up tests # will detect the db in error state L.exception('Forced to abort transaction on ZODB store opening', exc_info=True) transaction.abort() transaction.begin() self.graph.open(self.path)
def testMigrationTool(self): dfs_storage = self.__getDataFS() dfs_db = ZODB.DB(dfs_storage) self.__populate(dfs_db, with_undo=False) dump = self.__dump(dfs_storage) fs_path = dfs_storage.__name__ dfs_db.close() neo = self.neo neo.start() kw = {'cluster': neo.cluster_name, 'quiet': None} master_nodes = neo.master_nodes.replace('/', ' ') if neo.SSL: kw['ca'], kw['cert'], kw['key'] = neo.SSL p = NEOProcess('neomigrate', fs_path, master_nodes, **kw) p.start() p.wait() os.remove(fs_path) p = NEOProcess('neomigrate', master_nodes, fs_path, **kw) p.start() p.wait() self.assertEqual(dump, self.__dump(FileStorage(fs_path)))
def __init__(self, filename, realm=None): """ :param str filename: Config file with users and their password hashes :param str realm: ZODB's default permissions realm """ self.storage_filename = path.splitext(filename)[0] + ".db" self.storage = FileStorage.FileStorage(self.storage_filename) self.db = ZODB.DB(self.storage) self.db_conn = self.db.open() self.db_root = self.db_conn.root() root = self.db_root with transaction.manager: if not "users" in root: root["users"] = IdStore() # uid -> user if not "usernames" in root: root["usernames"] = self.family.OI.BTree() # username -> uid self.filename = filename self.load() # Frankly speaking, this realm-based security is questionable # Keep it here for now if realm: if self.realm and self.realm != realm: raise ValueError("Specified realm %r differs from database " "realm %r" % (realm or '', self.realm)) else: self.realm = realm self.noncekey = rand(32)
def setUp(self): # set up a zodb # we can't use DemoStorage here 'cos it doesn't support History self.dir = tempfile.mkdtemp() fs_path = os.path.join(self.dir, 'testHistory.fs') self.s = FileStorage(fs_path, create=True) self.connection = ZODB.DB(self.s).open() r = self.connection.root() a = Application() r['Application'] = a self.root = a # create a python script a['test'] = HistoryItem() self.hi = hi = a.test # commit some changes hi.title = 'First title' t = transaction.get() # undo note made by Application instantiation above. t.description = None t.note(u'Change 1') t.commit() time.sleep(0.02) # wait at least one Windows clock tick hi.title = 'Second title' t = transaction.get() t.note(u'Change 2') t.commit() time.sleep(0.02) # wait at least one Windows clock tick hi.title = 'Third title' t = transaction.get() t.note(u'Change 3') t.commit()
def __init__(self, file="data/work/Data.fs", open=1): self.db = ZODB.DB(ZODB.FileStorage.FileStorage(file)) if open == 1: self.connection = self.db.open() self.root = self.connection.root()
def get_root(): """ Return the root object of the current database. """ db_path = click.get_app_dir('katalog', force_posix=True) connection = ZODB.connection(db_path + '/data.fs') return connection.root()
def sandbox(base=None): '''Returns a sandbox copy of the base ZODB.''' if base is None: base = Zope2.DB base_storage = base._storage quota = getattr(base_storage, '_quota', None) storage = DemoStorage(base=base_storage, quota=quota) return ZODB.DB(storage)
def _start_child(zaddr): storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1) db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) setup(db.open()) conns = [] conn_count = 0 for i in range(NUM_CONNECTIONS): c = db.open() c.__count = 0 conns.append(c) conn_count += 1 while conn_count < 25: c = random.choice(conns) if c.__count > NUM_TRANSACTIONS_PER_CONN: conns.remove(c) c.close() conn_count += 1 c = db.open() c.__count = 0 conns.append(c) else: c.__count += 1 work(c)
def open(self): import ZODB from ZODB.FileStorage import FileStorage self.path = self.conf['rdf.store_conf'] openstr = os.path.abspath(self.path) try: fs = FileStorage(openstr) except IOError: L.exception("Failed to create a FileStorage") raise ZODBSourceOpenFailError(openstr) self.zdb = ZODB.DB(fs, cache_size=1600) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph('ZODB') self.graph = root['rdflib'] try: transaction.commit() except Exception: # catch commit exception and close db. # otherwise db would stay open and follow up tests # will detect the db in error state L.exception('Forced to abort transaction on ZODB store opening', exc_info=True) transaction.abort() transaction.begin() self.graph.open(self.path)
def main(): storage = ZODB.FileStorage.FileStorage("pupper.db") db = ZODB.DB(storage) with db.transaction() as c: for host in ("scc", "root", "puppettest5.scc.net.davepedu.com"): if "foo2" in c.root.nodes[host].classes: del c.root.nodes[host].classes["foo2"]
def setUp(self): self.__path = os.path.abspath('test.fs') store = ZODB.FileStorage.FileStorage(self.__path) self.db = ZODB.DB(store) warnings.filterwarnings('ignore', message='Versions are deprecated', module=__name__)
def __init__(self, storage_path): self.storage = ZODB.FileStorage.FileStorage(storage_path) self.database = ZODB.DB(self.storage) self.connection = self.database.open() self.root = self.connection.root try: self.root.scryfall_cards except (AttributeError, KeyError): self.root.scryfall_cards = PCardList() try: self.root.scryfall_sets except (AttributeError, KeyError): self.root.scryfall_sets = PSetList() try: self.root.mtgio_cards except (AttributeError, KeyError): self.root.mtgio_cards = PCardList() try: self.root.mtgio_sets except (AttributeError, KeyError): self.root.mtgio_sets = PSetList()
def setUp(self): # Our default transaction manager is # transaction._manager.ThreadTransactionManager # so no need to set it. ZODB.tests.util.TestCase.setUp(self) self._storage = ZODB.FileStorage.FileStorage('ZODBTests.fs', create=1) self._db = ZODB.DB(self._storage)
def getPeople(self, options): folder = tempfile.gettempdir() #folder = './' # my /tmp is a tmpfs fname = os.path.join(folder, 'performance_data.fs') if options.reload: try: os.remove(fname) except: pass fs = ZODB.FileStorage.FileStorage(fname) db = ZODB.DB(fs) conn = db.open() root = conn.root() if options.reload: root['people'] = people = PeopleZ() transaction.commit() # Profile inserts transaction.begin() t1 = time.time() for idx in xrange(options.size): klass = (self.personKlass if (MULTIPLE_CLASSES and idx % 2) else self.person2Klass) name = 'Mr Number %.5i' % idx people[name] = klass(name, random.randint(0, 100)) transaction.commit() t2 = time.time() self.printResult('Insert', t1, t2, options.size) else: people = root['people'] return people
def start(self): self.started = 1 self.path = tempfile.mktemp(suffix=".fs") self._storage = FileStorage(self.path) self.db = ZODB.DB(self._storage) self.do_updates() self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
def setUp(self): from ..site import Site self.site = Site("Test") self.generation = self.site.generation self.conn = ZODB.connection(None) self.conn.root.site = self.site self.conn.transaction_manager.commit()
class RN_Manager: storage = FS.FileStorage( os.path.join('SamurAI', 'database', 'IAs', 'index.fs')) db = ZODB.DB(storage) conn = db.open() dbroot = conn.root() ultimo_da_fila = dbroot['ultimo'] #if 'ultimo' in dbroot else 0 ias = dbroot['ias'] def __init__(self): pass @classmethod def proximo_indice(cls): if len(cls.ias) == MAX_RNs: return cls.ultimo_da_fila return len(cls.ias) @classmethod def registrar(cls, nome_arquivo): if len(cls.ias) == MAX_RNs: # registra o novo arquivo cls.ias[cls.ultimo_da_fila] = nome_arquivo # avança o ponteiro para o proximo da fila cls.ultimo_da_fila = (cls.ultimo_da_fila + 1) % MAX_RNs cls.dbroot['ultimo'] = cls.ultimo_da_fila else: cls.ias[len(cls.ias)] = nome_arquivo transaction.commit() @classmethod def close(cls): cls.conn.close() cls.db.close() cls.storage.close()
def open(self, databases=None): section = self.config storage = section.storage.open() options = {} def _option(name, oname=None): v = getattr(section, name) if v is not None: if oname is None: oname = name options[oname] = v _option('pool_timeout') _option('allow_implicit_cross_references', 'xrefs') _option('large_record_size') try: return ZODB.DB( storage, pool_size=section.pool_size, cache_size=section.cache_size, cache_size_bytes=section.cache_size_bytes, historical_pool_size=section.historical_pool_size, historical_cache_size=section.historical_cache_size, historical_cache_size_bytes=section.historical_cache_size_bytes, historical_timeout=section.historical_timeout, database_name=section.database_name or self.name or '', databases=databases, **options) except: storage.close() raise
def run(storage): if hasattr(storage, 'is_connected'): while not storage.is_connected(): time.sleep(CONNECT_DELAY) pid = os.getpid() print "Client process connected:", pid, storage db = ZODB.DB(storage) root = db.open().root() while 1: try: s = root[pid] = Stats() transaction.commit() except ZODB.POSException.ConflictError: transaction.abort() time.sleep(CONFLICT_DELAY) else: break dict = root["multi"] prev = None i = 0 while i < RECORDS_PER_CLIENT: try: size = len(dict) r = dict[size] = Record(pid, size) if prev: prev.set_next(r) transaction.commit() except ZODB.POSException.ConflictError, err: transaction.abort() time.sleep(CONFLICT_DELAY) else: i = i + 1 if VERBOSE and (i < 5 or i % 10 == 0): print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
def test_prefetch(self): db = ZODB.DB(None) fetched = [] def prefetch(oids, tid): fetched.append((list(map(u64, oids)), tid)) db.storage.prefetch = prefetch with db.transaction() as conn: for i in range(10): conn.root()[i] = conn.root().__class__() conn = db.open() conn.prefetch(z64) conn.prefetch([z64]) conn.prefetch(conn.root()) conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3]) self.assertEqual(fetched, [([0], conn._storage._start), ([0], conn._storage._start), ([0], conn._storage._start), ([0, 1, 2, 3, 4], conn._storage._start), ]) db.close()
def test_prefetch_optional_imvcc(self): conn = ZODB.connection(MVCCMappingStorage()) conn.prefetch(z64) conn.prefetch([z64]) conn.prefetch(conn.root()) conn.prefetch(z64, [z64]) conn.prefetch(z64, [z64], conn.root()) conn.close()
def test_prefetch_optional(self): conn = ZODB.connection(None) conn.prefetch(z64) conn.prefetch([z64]) conn.prefetch(conn.root()) conn.prefetch(z64, [z64]) conn.prefetch(z64, [z64], conn.root()) conn.close()
def opendb(self, contName=None): """ Open the database if not yet, return the required container. """ if self.conn is None: self.conn = ZODB.connection(self.db_path) if contName is None: contName = self.contName if contName is None: raise "must specify a container name" return self.getContainer(self.conn.root, contName)
def opendb(self): """ self.db is a dictionary-like object, the initial db will be empty, thus this statement: not self.db, will always yield a True, which cause the process try to open the DB again even if it did open it before, and cause an exception (locking problem). The correct way is compare it directly with None object. """ if self.db is None: connection = ZODB.connection(self.db_path) self.db = getContainer(connection.root, 'main')
def test_used_by_connection(self): import ZODB from ZODB.MappingStorage import MappingStorage class Storage(MappingStorage): def tpc_begin(self, transaction): self.test_transaction = transaction return MappingStorage.tpc_begin(self, transaction) storage = Storage() conn = ZODB.connection(storage) with conn.transaction_manager as t: t.user = u'user\x80' t.description = u'description\x80' t.setExtendedInfo('foo', 'FOO') conn.root.x = 1 t = storage.test_transaction self.assertEqual(t.__class__, TransactionMetaData) self.assertEqual(t.user, b'user\xc2\x80') self.assertEqual(t.description, b'description\xc2\x80') self.assertEqual(t.extension, dict(foo='FOO'))
def _open_db(path): conn = _connections.get(path) if not (conn and _db_is_opened(conn)): conn = ZODB.connection(path) _connections[path] = conn return conn