def __init__(self, db_name="d", host = None, port = None): self.model_has_been_created = False try: if not host and not port: self.storage = ZODB.FileStorage.FileStorage(file_name=db_name, blob_dir='.blob') self.db = ZODB.DB(self.storage) else: self.storage = ClientStorage.ClientStorage((host,port)) self.db = ZODB.DB(self.storage) self.connection = self.db.open() self.root = self.connection.root() _init(self) self.users = self.root['users'] self.piadas = self.root['piadas'] self.admin = self.root['admin'] pass except Exception as e: raise Exception("Database Error - " + str(e))
def db(self, db_name): """ Construct ZODB database if not existing, store DB object :param db_name: name of new db, including path """ is_existing_db = isfile(db_name) if is_existing_db: print("Opening db {db_name}".format(db_name=db_name)) storage = ZODB.FileStorage.FileStorage(db_name, read_only=True) self._db = ZODB.DB(storage) with self._db.transaction() as conn: self.width = conn.root.width self.nb_pos = conn.root.nb_pos self.nb_neg = conn.root.nb_neg #~ print(conn.root.neg[630309][0], conn.root.neg[630309][1]) #~ print(len(conn.root.neg[360354][0]), len(conn.root.neg[360354][1])) print("Width: ", self.width, "\t# pos: ", self.nb_pos, "\t# neg: ", self.nb_neg) else: storage = ZODB.FileStorage.FileStorage(db_name, read_only=False) self._db = ZODB.DB(storage) with self._db.transaction() as conn: conn.root.width = self.width conn.root.nb_pos = 0 conn.root.nb_neg = 0 conn.root.pos = BTrees.IOBTree.BTree() # IOB is for integer - is this about number or measurements - yes! conn.root.neg = BTrees.IOBTree.BTree()
def db(s): """ZODB connection pool. """ import atexit zuri = s.config.get('JOB_DB') if zuri is None: z = ZODB.DB("simsvc.fs") else: sf, kws = zodburi.resolve_uri(zuri) z = ZODB.DB(sf(), **kws) atexit.register(z.close) return z
def run(storage): if hasattr(storage, 'is_connected'): while not storage.is_connected(): time.sleep(CONNECT_DELAY) pid = os.getpid() print "Client process connected:", pid, storage db = ZODB.DB(storage) root = db.open().root() while 1: try: s = root[pid] = Stats() transaction.commit() except ZODB.POSException.ConflictError: transaction.abort() time.sleep(CONFLICT_DELAY) else: break dict = root["multi"] prev = None i = 0 while i < RECORDS_PER_CLIENT: try: size = len(dict) r = dict[size] = Record(pid, size) if prev: prev.set_next(r) transaction.commit() except ZODB.POSException.ConflictError, err: transaction.abort() time.sleep(CONFLICT_DELAY) else: i = i + 1 if VERBOSE and (i < 5 or i % 10 == 0): print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
def test_prefetch(self): db = ZODB.DB(None) fetched = [] def prefetch(oids, tid): fetched.append((list(map(u64, oids)), tid)) db.storage.prefetch = prefetch with db.transaction() as conn: for i in range(10): conn.root()[i] = conn.root().__class__() conn = db.open() conn.prefetch(z64) conn.prefetch([z64]) conn.prefetch(conn.root()) conn.prefetch(z64, (conn.root()[i] for i in range(3)), conn.root()[3]) self.assertEqual(fetched, [([0], conn._storage._start), ([0], conn._storage._start), ([0], conn._storage._start), ([0, 1, 2, 3, 4], conn._storage._start), ]) db.close()
def on_post(self, req, resp, token): trello = TrelloApi(trelloconfig.api_key, token=trelloconfig.token) trello_token = trello.get_token_url('My App', expires='30days', write_access=True) storage = ZODB.FileStorage.FileStorage('trees/' + token + '.fs') db = ZODB.DB(storage) connection = db.open() root = connection.root if hasattr(root, 'tree'): tree = root.tree else: resp.body = "Initialize first" connection.close() db.close() storage.close() return lst = list(btree.inorder(tree)) connection.close() db.close() storage.close() if len(lst) > 0: id_new = trello.boards.new_list('Le5vKw7H', token)['id'] for card in lst: trello.cards.new(card, id_new)
def getPeople(self, options): folder = tempfile.gettempdir() #folder = './' # my /tmp is a tmpfs fname = os.path.join(folder, 'performance_data.fs') if options.reload: try: os.remove(fname) except: pass fs = ZODB.FileStorage.FileStorage(fname) db = ZODB.DB(fs) conn = db.open() root = conn.root() if options.reload: root['people'] = people = PeopleZ() transaction.commit() # Profile inserts transaction.begin() t1 = time.time() for idx in xrange(options.size): klass = (self.personKlass if (MULTIPLE_CLASSES and idx % 2) else self.person2Klass) name = 'Mr Number %.5i' % idx people[name] = klass(name, random.randint(0, 100)) transaction.commit() t2 = time.time() self.printResult('Insert', t1, t2, options.size) else: people = root['people'] return people
def _start_child(zaddr): storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1) db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS) setup(db.open()) conns = [] conn_count = 0 for i in range(NUM_CONNECTIONS): c = db.open() c.__count = 0 conns.append(c) conn_count += 1 while conn_count < 25: c = random.choice(conns) if c.__count > NUM_TRANSACTIONS_PER_CONN: conns.remove(c) c.close() conn_count += 1 c = db.open() c.__count = 0 conns.append(c) else: c.__count += 1 work(c)
def setUp(self): self.__path = os.path.abspath('test.fs') store = ZODB.FileStorage.FileStorage(self.__path) self.db = ZODB.DB(store) warnings.filterwarnings('ignore', message='Versions are deprecated', module=__name__)
def _get_connection(cls): try: return cls.Meta.connection except AttributeError: zodb = ZODB.DB(None) cls.Meta.connection = zodb.open() return cls.Meta.connection
def start(self): self.started = 1 self.path = tempfile.mktemp(suffix=".fs") self._storage = FileStorage(self.path) self.db = ZODB.DB(self._storage) self.do_updates() self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
def sandbox(base=None): '''Returns a sandbox copy of the base ZODB.''' if base is None: base = Zope2.DB base_storage = base._storage quota = getattr(base_storage, '_quota', None) storage = DemoStorage(base=base_storage, quota=quota) return ZODB.DB(storage)
def get_zodb_root(): global __zodb_root if not __zodb_root: db = ZODB.DB('data/db.fs') connection = db.open() __zodb_root = connection.root return __zodb_root
def get_stored_structure_data(): try: f = FfAA.Settings.file_for_storing_structures d = os.path.split(f)[0] if not os.path.exists(d): print "Creating directory", d os.mkdir(d) storage = ZODB.FileStorage.FileStorage(f) db = ZODB.DB(storage) conn = db.open() dbroot = conn.root() if not dbroot.has_key('structures'): print_color(3, "Add structures to database\n") dbroot['structures'] = Structures() structures = dbroot['structures'] init = Initializer(structures) except ZODB.POSException.StorageSystemError: print "Sound Structure database already opened by another process." db = structures = init = None return db, structures, init
def setUp(self): # set up a zodb # we can't use DemoStorage here 'cos it doesn't support History self.dir = tempfile.mkdtemp() self.s = FileStorage(os.path.join(self.dir, 'testHistory.fs'), create=True) self.connection = ZODB.DB(self.s).open() r = self.connection.root() a = Application() r['Application'] = a self.root = a # create a python script manage_addPythonScript(a, 'test') self.ps = ps = a.test # commit some changes ps.write('return 1') t = transaction.get() # undo note made by Application instantiation above. t.description = None t.note('Change 1') t.commit() ps.write('return 2') t = transaction.get() t.note('Change 2') t.commit() ps.write('return 3') t = transaction.get() t.note('Change 3') t.commit()
def testExport(self): # create a neo storage self.neo.start() (neo_db, neo_conn) = self.neo.getZODBConnection() self.__populate(neo_db) dump = self.__dump(neo_db.storage) # copy neo to data fs dfs_storage = self.__getDataFS(reset=True) neo_storage = self.neo.getZODBStorage() dfs_storage.copyTransactionsFrom(neo_storage) # check data fs content dfs_db = ZODB.DB(dfs_storage) root = dfs_db.open().root() self.__checkTree(root['trees']) dfs_db.close() self.neo.stop() self.neo = NEOCluster(db_list=['test_neo1'], partitions=3, importer=[("root", { "storage": "<filestorage>\npath %s\n</filestorage>" % dfs_storage.getName() })], temp_dir=self.getTempDirectory()) self.neo.start() neo_db, neo_conn = self.neo.getZODBConnection() self.__checkTree(neo_conn.root()['trees']) self.assertEqual(dump, self.__dump(neo_db.storage))
def open_database(self): """Create the internal engine and session builder for this manager based on the configured database""" if self.sessions and self.db is not None: raise RuntimeError( "cannot change database after sessions are established") # Connect/open the database factory_class, factory_args = zodburi.resolve_uri(self.config["db"]) storage = factory_class() self.db = ZODB.DB(storage, **factory_args) conn = self.db.open() if not hasattr(conn.root, "targets"): conn.root.targets = persistent.list.PersistentList() if not hasattr(conn.root, "history"): conn.root.history = persistent.list.PersistentList() conn.transaction_manager.commit() conn.close() # Rebuild the command parser now that the database is available self.parser = CommandParser(self)
def setUp(self): # Our default transaction manager is # transaction._manager.ThreadTransactionManager # so no need to set it. ZODB.tests.util.TestCase.setUp(self) self._storage = ZODB.FileStorage.FileStorage('ZODBTests.fs', create=1) self._db = ZODB.DB(self._storage)
class RN_Manager: storage = FS.FileStorage( os.path.join('SamurAI', 'database', 'IAs', 'index.fs')) db = ZODB.DB(storage) conn = db.open() dbroot = conn.root() ultimo_da_fila = dbroot['ultimo'] #if 'ultimo' in dbroot else 0 ias = dbroot['ias'] def __init__(self): pass @classmethod def proximo_indice(cls): if len(cls.ias) == MAX_RNs: return cls.ultimo_da_fila return len(cls.ias) @classmethod def registrar(cls, nome_arquivo): if len(cls.ias) == MAX_RNs: # registra o novo arquivo cls.ias[cls.ultimo_da_fila] = nome_arquivo # avança o ponteiro para o proximo da fila cls.ultimo_da_fila = (cls.ultimo_da_fila + 1) % MAX_RNs cls.dbroot['ultimo'] = cls.ultimo_da_fila else: cls.ias[len(cls.ias)] = nome_arquivo transaction.commit() @classmethod def close(cls): cls.conn.close() cls.db.close() cls.storage.close()
def get_N_urls(main_url, N): try: storage = ZODB.FileStorage.FileStorage('data.fs') db = ZODB.DB(storage) connection = db.open() root = connection.root except: # Невозможно получить данные, предварительно не скачав их db.close() print("Cannot 'get' before 'load'") return if main_url in root.urls: l = list(root.urls[main_url][1]) else: # Когда запрашиваемой ссылки нет в базе. Это значит, что ссылка # принадлежит к третьему уровню вложенности (не обрабатывалась) # или такой ссылки вообще не было в загруженных данных l = [] db.close() print("No such URL in the DB") return if N > len(l): N = len(l) for i in range(N): url = l[i] try: title = root.urls[url][0] except: # Ссылка третьего уровня - для неё нет данных кроме самого URL title = "Not found in DB" print(url + ': "' + title + '"') db.close()
def __init__(self, storage_path): self.storage = ZODB.FileStorage.FileStorage(storage_path) self.database = ZODB.DB(self.storage) self.connection = self.database.open() self.root = self.connection.root try: self.root.scryfall_cards except (AttributeError, KeyError): self.root.scryfall_cards = PCardList() try: self.root.scryfall_sets except (AttributeError, KeyError): self.root.scryfall_sets = PSetList() try: self.root.mtgio_cards except (AttributeError, KeyError): self.root.mtgio_cards = PCardList() try: self.root.mtgio_sets except (AttributeError, KeyError): self.root.mtgio_sets = PSetList()
def __init__(self, file="data/work/Data.fs", open=1): self.db = ZODB.DB(ZODB.FileStorage.FileStorage(file)) if open == 1: self.connection = self.db.open() self.root = self.connection.root()
def main(): storage = ZODB.FileStorage.FileStorage("pupper.db") db = ZODB.DB(storage) with db.transaction() as c: for host in ("scc", "root", "puppettest5.scc.net.davepedu.com"): if "foo2" in c.root.nodes[host].classes: del c.root.nodes[host].classes["foo2"]
def setUp(self): # set up a zodb # we can't use DemoStorage here 'cos it doesn't support History self.dir = tempfile.mkdtemp() fs_path = os.path.join(self.dir, 'testHistory.fs') self.s = FileStorage(fs_path, create=True) self.connection = ZODB.DB(self.s).open() r = self.connection.root() a = Application() r['Application'] = a self.root = a # create a python script a['test'] = HistoryItem() self.hi = hi = a.test # commit some changes hi.title = 'First title' t = transaction.get() # undo note made by Application instantiation above. t.description = None t.note(u'Change 1') t.commit() time.sleep(0.02) # wait at least one Windows clock tick hi.title = 'Second title' t = transaction.get() t.note(u'Change 2') t.commit() time.sleep(0.02) # wait at least one Windows clock tick hi.title = 'Third title' t = transaction.get() t.note(u'Change 3') t.commit()
def open(self): import ZODB from ZODB.FileStorage import FileStorage self.path = self.conf['rdf.store_conf'] openstr = os.path.abspath(self.path) try: fs = FileStorage(openstr) except IOError: L.exception("Failed to create a FileStorage") raise ZODBSourceOpenFailError(openstr) self.zdb = ZODB.DB(fs, cache_size=1600) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph('ZODB') self.graph = root['rdflib'] try: transaction.commit() except Exception: # catch commit exception and close db. # otherwise db would stay open and follow up tests # will detect the db in error state L.exception('Forced to abort transaction on ZODB store opening', exc_info=True) transaction.abort() transaction.begin() self.graph.open(self.path)
def __init__(self, filename, realm=None): """ :param str filename: Config file with users and their password hashes :param str realm: ZODB's default permissions realm """ self.storage_filename = path.splitext(filename)[0] + ".db" self.storage = FileStorage.FileStorage(self.storage_filename) self.db = ZODB.DB(self.storage) self.db_conn = self.db.open() self.db_root = self.db_conn.root() root = self.db_root with transaction.manager: if not "users" in root: root["users"] = IdStore() # uid -> user if not "usernames" in root: root["usernames"] = self.family.OI.BTree() # username -> uid self.filename = filename self.load() # Frankly speaking, this realm-based security is questionable # Keep it here for now if realm: if self.realm and self.realm != realm: raise ValueError("Specified realm %r differs from database " "realm %r" % (realm or '', self.realm)) else: self.realm = realm self.noncekey = rand(32)
def open(self, databases=None): section = self.config storage = section.storage.open() options = {} def _option(name, oname=None): v = getattr(section, name) if v is not None: if oname is None: oname = name options[oname] = v _option('pool_timeout') _option('allow_implicit_cross_references', 'xrefs') _option('large_record_size') try: return ZODB.DB( storage, pool_size=section.pool_size, cache_size=section.cache_size, cache_size_bytes=section.cache_size_bytes, historical_pool_size=section.historical_pool_size, historical_cache_size=section.historical_cache_size, historical_cache_size_bytes=section.historical_cache_size_bytes, historical_timeout=section.historical_timeout, database_name=section.database_name or self.name or '', databases=databases, **options) except: storage.close() raise
def testMigrationTool(self): dfs_storage = self.__getDataFS() dfs_db = ZODB.DB(dfs_storage) self.__populate(dfs_db, with_undo=False) dump = self.__dump(dfs_storage) fs_path = dfs_storage.__name__ dfs_db.close() neo = self.neo neo.start() kw = {'cluster': neo.cluster_name, 'quiet': None} master_nodes = neo.master_nodes.replace('/', ' ') if neo.SSL: kw['ca'], kw['cert'], kw['key'] = neo.SSL p = NEOProcess('neomigrate', fs_path, master_nodes, **kw) p.start() p.wait() os.remove(fs_path) p = NEOProcess('neomigrate', master_nodes, fs_path, **kw) p.start() p.wait() self.assertEqual(dump, self.__dump(FileStorage(fs_path)))
def setUp(self): if self.url.endswith('.fs'): from ZODB.FileStorage import FileStorage if os.path.exists(self.path): os.unlink('/tmp/zodb_local3.fs') os.unlink('/tmp/zodb_local3.fs.index') os.unlink('/tmp/zodb_local3.fs.tmp') os.unlink('/tmp/zodb_local3.fs.lock') openstr = os.path.abspath(os.path.expanduser(self.url[7:])) fs = FileStorage(openstr) else: from ZEO.ClientStorage import ClientStorage schema, opts = _parse_rfc1738_args(self.url) fs = ClientStorage((opts['host'], int(opts['port']))) self.zdb = ZODB.DB(fs) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph(self.store_name) self.graph = self.g = root['rdflib'] self.michel = URIRef('michel') self.tarek = URIRef('tarek') self.bob = URIRef('bob') self.likes = URIRef('likes') self.hates = URIRef('hates') self.pizza = URIRef('pizza') self.cheese = URIRef('cheese') transaction.commit()
def open(self): import ZODB from ZODB.FileStorage import FileStorage from zc.lockfile import LockError self.path = self.conf['rdf.store_conf'] openstr = os.path.abspath(self.path) try: fs = FileStorage(openstr) except IOError: L.exception("Failed to create a FileStorage") raise ZODBSourceOpenFailError(openstr) except LockError: L.exception('Found database "{}" is locked when trying to open it. ' 'The PID of this process: {}'.format(openstr, os.getpid()), exc_info=True) raise DatabaseConflict('Database ' + openstr + ' locked') self.zdb = ZODB.DB(fs, cache_size=1600) self.conn = self.zdb.open() root = self.conn.root() if 'rdflib' not in root: root['rdflib'] = ConjunctiveGraph('ZODB') self.graph = root['rdflib'] try: transaction.commit() except Exception: # catch commit exception and close db. # otherwise db would stay open and follow up tests # will detect the db in error state L.exception('Forced to abort transaction on ZODB store opening', exc_info=True) transaction.abort() transaction.begin() self.graph.open(self.path)