def test_database_integration_because_ints(self):
from zodburi import resolve_uri
self.tmp.write(b"""
<zodb>
<mappingstorage>
</mappingstorage>
</zodb>
""")
self.tmp.flush()
from zodburi import resolve_uri
factory, dbkw = resolve_uri('zconfig://%s' % self.tmp.name)
storage = factory()
from ZODB.MappingStorage import MappingStorage
self.assertTrue(isinstance(storage, MappingStorage))
if ZODB_VERSION[0] < 5:
self.assertEqual(dbkw,
{'cache_size': 5000,
'cache_size_bytes': 0,
'historical_cache_size': 1000,
'historical_cache_size_bytes': 0,
'historical_pool_size': 3,
'historical_timeout': 300,
'pool_size': 7,
'database_name': 'unnamed'})
else:
self.assertEqual(dbkw,
{'cache_size': 5000,
'cache_size_bytes': 0,
'historical_cache_size': 1000,
'historical_cache_size_bytes': 0,
'historical_pool_size': 3,
'historical_timeout': 300,
'large_record_size': 16777216,
'pool_size': 7,
'database_name': 'unnamed'})
def reset(self, storage=None, uri=None):
"""Close and reopen a database connection."""
if self.db:
self.close()
if storage is not None:
self.storage = storage
elif uri:
storage_factory, db_args = zodburi.resolve_uri(uri)
self.storage = storage_factory()
db_args.update(self.db_args)
self.db_args = db_args
else:
self.storage = DemoStorage()
self.name = self.db_args.get("database_name",
Database.DEFAULT_DATABASE_NAME)
if self.name in LocalData.get().databases:
last_context = LocalData.get().last_database_context.get(self.name)
raise KeyError(
"A database named '{}' already exists. Last opening was on {} at line {}"
.format(self.name, last_context[0], last_context[1])
if last_context else "A database named '{}' already exists.".
format(self.name))
self.db = ZODB.DB(self.storage, **self.db_args)
LocalData.get().databases[self.name] = self
def _get_zodb_database(config: ConfigParser) -> dict: # pragma: no cover
config_locations = config['app:main']['yaml.location']
if not isinstance(config_locations, (list, tuple)):
config_locations = config_locations.split(',')
env = config['app:main'].get('env', 'dev')
file_paths = []
files = ['config.yaml', 'config.yml']
for location in config_locations:
path = _translate_config_path(location)
current_files = files
if os.path.isfile(path):
path, current_files = os.path.split(path)
current_files = [current_files]
for config_path in [
os.path.join(path, f)
for f in _env_filenames(current_files, env)
]:
if os.path.isfile(config_path):
file_paths.append(config_path)
zodb_uri = ''
for config_file in file_paths:
with open(config_file, 'r') as stream:
yaml_config = yaml.load(stream)
configurator = yaml_config.get('configurator') or {}
zodbconn = configurator.get('zodbconn') or {}
new_zodb_uri = zodbconn.get('uri')
if new_zodb_uri:
zodb_uri = new_zodb_uri
logger.info('Getting ZEO database on {}'.format(zodb_uri))
storage_factory, dbkw = resolve_uri(zodb_uri)
storage = storage_factory()
db = DB(storage, **dbkw)
return db
def __init__(self):
# Setup DB
storage = zodburi.resolve_uri(config.database_url)[0]()
self._db = ZODB.DB(storage)
self._connection = self._db.open()
self.root = self._connection.root
self.migrate()
def test_database_integration_because_ints(self):
from zodburi import resolve_uri
self.tmp.write(b"""
<zodb>
<mappingstorage>
</mappingstorage>
</zodb>
""")
self.tmp.flush()
from zodburi import resolve_uri
factory, dbkw = resolve_uri('zconfig://%s' % self.tmp.name)
storage = factory()
from ZODB.MappingStorage import MappingStorage
self.assertTrue(isinstance(storage, MappingStorage))
self.assertEqual(
dbkw, {
'cache_size': 5000,
'cache_size_bytes': 0,
'historical_cache_size': 1000,
'historical_cache_size_bytes': 0,
'historical_pool_size': 3,
'historical_timeout': 300,
'large_record_size': 16777216,
'pool_size': 7,
'database_name': 'unnamed'
})
def fork_zeo(self):
self.zeo_path = get_socket_addr('zeo')
try:
os.unlink(self.zeo_path)
except OSError:
pass
pid = self.fork()
if pid:
return pid
else:
from ZEO.StorageServer import StorageServer
set_process_name('borgcubed [database process]')
storage_factory, _ = zodburi.resolve_uri(settings.DB_URI)
storage = storage_factory()
prev_umask = os.umask(0o177)
server = StorageServer(
addr=self.zeo_path,
storages={'1': storage},
)
os.umask(prev_umask)
os.kill(os.getppid(), signal.SIGUSR1)
try:
server.loop()
finally:
server.close()
sys.exit(0)
def _get_zodb_database(config: ConfigParser) -> dict:
zodb_uri = config['app:main']['zodbconn.uri']
logger.info('Getting ZEO database on {}'.format(zodb_uri))
storage_factory, dbkw = resolve_uri(zodb_uri)
storage = storage_factory()
db = DB(storage, **dbkw)
return db
def open_database(self):
"""Create the internal engine and session builder
for this manager based on the configured database"""
if self.sessions and self.db is not None:
raise RuntimeError(
"cannot change database after sessions are established")
# Connect/open the database
factory_class, factory_args = zodburi.resolve_uri(self.config["db"])
storage = factory_class()
self.db = ZODB.DB(storage, **factory_args)
conn = self.db.open()
if not hasattr(conn.root, "targets"):
conn.root.targets = persistent.list.PersistentList()
if not hasattr(conn.root, "history"):
conn.root.history = persistent.list.PersistentList()
conn.transaction_manager.commit()
conn.close()
# Rebuild the command parser now that the database is available
self.parser = CommandParser(self)
def test_it(self, MappingStorage):
from zodburi import resolve_uri
factory, dbkw = resolve_uri('memory://')
factory()
MappingStorage.assert_called_once_with('')
self.assertEqual(dbkw, {
'cache_size': 10000,
'pool_size': 7,
'database_name': 'unnamed'})
def test_it(self, MappingStorage):
from zodburi import resolve_uri
factory, dbkw = resolve_uri('memory://')
factory()
MappingStorage.assert_called_once_with('')
self.assertEqual(dbkw, {
'cache_size': 10000,
'pool_size': 7,
'database_name': 'unnamed'})
def get_persistent_settings(self):
uri = self.settings['zodbconn.uri']
storage_factory, dbkw = resolve_uri(uri)
db = DB(storage_factory(), **dbkw)
try:
conn = db.open()
home = self.find_home(conn.root())
return home['settings']
finally:
db.close()
def initializeDB(self):
# Get the database
# See: http://docs.pylonsproject.org/projects/zodburi/en/latest/
# storage = ZODB.FileStorage.FileStorage(os.path.join(dirPrefix,"overwatch.fs"))
storage_factory, dbArgs = zodburi.resolve_uri(self.databaseLocation)
storage = storage_factory()
db = ZODB.DB(storage, **dbArgs)
connection = db.open()
dbRoot = connection.root()
return ZodbDatabase(dbRoot, connection)
def getDB(databaseLocation):
# Get the database
# See: http://docs.pylonsproject.org/projects/zodburi/en/latest/
#storage = ZODB.FileStorage.FileStorage(os.path.join(dirPrefix,"overwatch.fs"))
storage_factory, dbArgs = zodburi.resolve_uri(databaseLocation)
storage = storage_factory()
db = ZODB.DB(storage, **dbArgs)
connection = db.open()
dbRoot = connection.root()
return (dbRoot, connection)
def test_it_with_dbkw(self, MappingStorage):
from zodburi import resolve_uri
factory, dbkw = resolve_uri(
'memory://test?connection_cache_size=1&connection_pool_size=2&'
'database_name=dbname')
factory()
MappingStorage.assert_called_once_with('test')
self.assertEqual(dbkw, {
'cache_size': 1,
'pool_size': 2,
'database_name': 'dbname'})
def create_db(self, app):
"""Create a ZODB connection pool from the *app* configuration."""
assert 'ZODB_STORAGE' in app.config, \
'ZODB_STORAGE not configured'
storage = app.config['ZODB_STORAGE']
if isinstance(storage, basestring):
factory, dbargs = zodburi.resolve_uri(storage)
elif isinstance(storage, tuple):
factory, dbargs = storage
else:
factory, dbargs = storage, {}
return DB(factory(), **dbargs)
def db(s):
"""ZODB connection pool.
"""
import atexit
zuri = s.config.get('JOB_DB')
if zuri is None:
z = ZODB.DB("simsvc.fs")
else:
sf, kws = zodburi.resolve_uri(zuri)
z = ZODB.DB(sf(), **kws)
atexit.register(z.close)
return z
def create_db(self, app):
"""Create a ZODB connection pool from the *app* configuration."""
assert 'ZODB_STORAGE' in app.config, \
'ZODB_STORAGE not configured'
storage = app.config['ZODB_STORAGE']
if isinstance(storage, basestring):
factory, dbargs = zodburi.resolve_uri(storage)
elif isinstance(storage, tuple):
factory, dbargs = storage
else:
factory, dbargs = storage, {}
return DB(factory(), **dbargs)
def db():
global _db
if not _db:
with _db_lock:
storage_factory, dbkw = zodburi.resolve_uri(settings.DB_URI)
storage = storage_factory()
_db = DB(storage, **dbkw)
return db()
try:
return _db_local.conn
except AttributeError:
log.debug('Opening new database connection')
_db_local.conn = _db.open()
return _db_local.conn
def test_it_with_dbkw(self, MappingStorage):
from zodburi import resolve_uri, connection_parameters, parameters
uri = 'memory://test?database_name=dbname'
for i, parameter in enumerate(connection_parameters):
uri += '&connection_%s=%d' % (parameter, i)
if parameter == 'cache_size_bytes':
uri += 'MB'
factory, dbkw = resolve_uri(uri)
factory()
MappingStorage.assert_called_once_with('test')
expect = dict(database_name='dbname')
for i, parameter in enumerate(connection_parameters):
parameter = 'connection_' + parameter
expect[parameters[parameter]] = i
if parameter == 'connection_cache_size_bytes':
expect[parameters[parameter]] *= 1 << 20
self.assertEqual(dbkw, expect)
def test_it_with_dbkw(self, MappingStorage):
from zodburi import resolve_uri, connection_parameters, parameters
uri = 'memory://test?database_name=dbname'
for i, parameter in enumerate(connection_parameters):
uri += '&connection_%s=%d' % (parameter, i)
if parameter == 'cache_size_bytes':
uri += 'MB'
factory, dbkw = resolve_uri(uri)
factory()
MappingStorage.assert_called_once_with('test')
expect = dict(database_name='dbname')
for i, parameter in enumerate(connection_parameters):
parameter = 'connection_' + parameter
expect[parameters[parameter]] = i
if parameter == 'connection_cache_size_bytes':
expect[parameters[parameter]] *= 1<<20
self.assertEqual(dbkw, expect)
def create_db(self, app):
"""Create a ZODB connection pool from the *app* configuration."""
assert 'ZODB_STORAGE' in app.config, \
'ZODB_STORAGE not configured'
storage = app.config['ZODB_STORAGE']
def storage_instance(obj):
"""Python 2.x and 3 compatiblity"""
try:
return isinstance(obj, basestring)
except NameError:
return isinstance(obj, str)
if storage_instance(storage):
factory, dbargs = zodburi.resolve_uri(storage)
elif isinstance(storage, tuple):
factory, dbargs = storage
else:
factory, dbargs = storage, {}
return DB(factory(), **dbargs)
def getDB(databaseLocation):
""" Setup and retrieve the database available at the given location.
Args:
databaseLocation (str): Path to the database. Must be a valid zodburi URI, which could be a local
file, a socket, a network path, or another type.
Returns:
tuple: (ZODB db root PersistentMapping, ZODB.Connection.Connection object). The connection object
should be closed work with the database is completed.
"""
# Get the database
# See: http://docs.pylonsproject.org/projects/zodburi/en/latest/
#storage = ZODB.FileStorage.FileStorage(os.path.join(dirPrefix,"overwatch.fs"))
storage_factory, dbArgs = zodburi.resolve_uri(databaseLocation)
storage = storage_factory()
db = ZODB.DB(storage, **dbArgs)
connection = db.open()
dbRoot = connection.root()
return (dbRoot, connection)
def storageFromURL(url, read_only=None):
# no schema -> file://
if "://" not in url:
url = "file://" + url
# read_only -> url
if read_only is not None:
scheme, netloc, path, query, fragment = urlsplit(url)
# XXX this won't have effect with zconfig:// but for file:// neo://
# zeo:// etc ... it works
if scheme != "zconfig":
if len(query) > 0:
query += "&"
query += "read_only=%s" % read_only
url = urlunsplit((scheme, netloc, path, query, fragment))
stor_factory, dbkw = zodburi.resolve_uri(url)
stor = stor_factory()
return stor
def db_from_uri(uri, resolve_uri=resolve_uri):
storage_factory, dbkw = resolve_uri(uri)
storage = storage_factory()
return DB(storage, **dbkw)
#!/usr/bin/env python from zodburi import resolve_uri from ZODB.DB import DB zodb_URI = 'zeo://console:1234' storage_factory, dbkw = resolve_uri(zodb_URI) storage = storage_factory() db = DB(storage, **dbkw) conn = db.open() dbroot = conn.root() print dbroot.keys() groups = dbroot['sim_groups'] print groups choices = [(group, group) for group in groups.keys()] print choices
#!/usr/bin/env python
'''
Clear a zodb database.
'''
import sys
import transaction
from ZODB import DB
from zodburi import resolve_uri
from ZODB.POSException import ConflictError
if len(sys.argv) < 2:
print 'Nothing has been done since no ZODB database provided!'
sys.exit()
storage_factory, dbkw = resolve_uri(sys.argv[1])
storage = storage_factory()
db = DB(storage, **dbkw)
conn = db.open()
root = conn.root()
t = 0
if len(root.keys()) > 0:
retry = 0
for k in root.keys():
while retry < 10:
try:
del root[k]
transaction.commit()
except ConflictError:
def db_from_uri(uri):
storage_factory, dbkw = resolve_uri(uri)
return DB(storage_factory(), **dbkw)
def db_from_uri(uri, dbname, dbmap, resolve_uri=resolve_uri):
storage_factory, dbkw = resolve_uri(uri)
dbkw['database_name'] = dbname
storage = storage_factory()
return DB(storage, databases=dbmap, **dbkw)
def connect_zodb(zodb_URI):
storage_factory, dbkw = resolve_uri(zodb_URI)
storage = storage_factory()
db = DB(storage, **dbkw)
conn = db.open()
return conn.root()
def db_from_uri(uri, dbname, dbmap, resolve_uri=resolve_uri):
storage_factory, dbkw = resolve_uri(uri)
dbkw['database_name'] = dbname
storage = storage_factory()
return DB(storage, databases=dbmap, **dbkw)
def db_from_uri(uri: str, name: str, databases: dict):
factory, params = resolve_uri(uri)
params['database_name'] = name
storage = factory()
return DB(storage, databases=databases, **params)
def make_init_db(uri):
storage_factory, dbkw = resolve_uri(uri)
storage = storage_factory()
db = DB(storage, **dbkw)
return db
def open(self):
storage_factory, dbkw = resolve_uri(self.uri)
storage = storage_factory()
self.db = DB(storage, **dbkw)
self.conn = self.db.open()
transaction.begin()
