def setUp(self):
if self.url.endswith('.fs'):
from ZODB.FileStorage import FileStorage
if os.path.exists(self.path):
os.unlink('/tmp/zodb_local3.fs')
os.unlink('/tmp/zodb_local3.fs.index')
os.unlink('/tmp/zodb_local3.fs.tmp')
os.unlink('/tmp/zodb_local3.fs.lock')
openstr = os.path.abspath(os.path.expanduser(self.url[7:]))
fs = FileStorage(openstr)
else:
from ZEO.ClientStorage import ClientStorage
schema, opts = _parse_rfc1738_args(self.url)
fs = ClientStorage((opts['host'], int(opts['port'])))
self.zdb = ZODB.DB(fs)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph(self.store_name)
self.graph = self.g = root['rdflib']
self.michel = URIRef('michel')
self.tarek = URIRef('tarek')
self.bob = URIRef('bob')
self.likes = URIRef('likes')
self.hates = URIRef('hates')
self.pizza = URIRef('pizza')
self.cheese = URIRef('cheese')
transaction.commit()
def checkZEOInvalidation(self):
addr = self._storage._addr
storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
try:
oid = self._storage.new_oid()
ob = MinPO('first')
revid1 = self._dostore(oid, data=ob)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('first'))
self.assertEqual(serial, revid1)
revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
# Now, storage 2 should eventually get the new data. It
# will take some time, although hopefully not much.
# We'll poll till we get it and whine if we time out:
for n in range(30):
time.sleep(.1)
data, serial = storage2.load(oid, '')
if (serial == revid2
and zodb_unpickle(data) == MinPO('second')):
break
else:
raise AssertionError('Invalidation message was not sent!')
finally:
storage2.close()
def open(self):
from ZEO.ClientStorage import ClientStorage
# config.server is a multikey of socket-connection-address values
# where the value is a socket family, address tuple.
config = self.config
addresses = [server.address for server in config.server]
options = {}
if config.blob_cache_size is not None:
options['blob_cache_size'] = config.blob_cache_size
if config.blob_cache_size_check is not None:
options['blob_cache_size_check'] = config.blob_cache_size_check
if config.client_label is not None:
options['client_label'] = config.client_label
ssl = config.ssl
if ssl:
options['ssl'] = ssl[0]
options['ssl_server_hostname'] = ssl[1]
return ClientStorage(addresses,
blob_dir=config.blob_dir,
shared_blob_dir=config.shared_blob_dir,
storage=config.storage,
cache_size=config.cache_size,
cache=config.cache_path,
name=config.name,
read_only=config.read_only,
read_only_fallback=config.read_only_fallback,
server_sync=config.server_sync,
wait_timeout=config.wait_timeout,
**options)
def getStorage():
""" Return a storage instance for running ZopeTestCase based
tests. By default a DemoStorage is used. Setting
$TEST_ZEO_HOST/TEST_ZEO_PORT environment variables allows you
to use a ZEO server instead. A file storage can be configured
by settting the $TEST_FILESTORAGE environment variable.
"""
get = os.environ.get
if os.environ.has_key('TEST_ZEO_HOST') and os.environ.has_key(
'TEST_ZEO_PORT'):
from ZEO.ClientStorage import ClientStorage
zeo_host = get('TEST_ZEO_HOST')
zeo_port = int(get('TEST_ZEO_PORT'))
LOG.info('Using ZEO server (%s:%d)' % (zeo_host, zeo_port))
return ClientStorage((zeo_host, zeo_port))
elif os.environ.has_key('TEST_FILESTORAGE'):
import ZODB.FileStorage
datafs = get('TEST_FILESTORAGE')
LOG.info('Using Filestorage at (%s)' % datafs)
return ZODB.FileStorage.FileStorage(datafs)
else:
from ZODB.DemoStorage import DemoStorage
LOG.info('Using DemoStorage')
return DemoStorage()
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
def open(self):
from ZEO.ClientStorage import ClientStorage
# config.server is a multikey of socket-connection-address values
# where the value is a socket family, address tuple.
L = [server.address for server in self.config.server]
options = {}
if self.config.blob_cache_size is not None:
options['blob_cache_size'] = self.config.blob_cache_size
if self.config.blob_cache_size_check is not None:
options['blob_cache_size_check'] = self.config.blob_cache_size_check
if self.config.client_label is not None:
options['client_label'] = self.config.client_label
return ClientStorage(
L,
blob_dir=self.config.blob_dir,
shared_blob_dir=self.config.shared_blob_dir,
storage=self.config.storage,
cache_size=self.config.cache_size,
name=self.config.name,
client=self.config.client,
var=self.config.var,
min_disconnect_poll=self.config.min_disconnect_poll,
max_disconnect_poll=self.config.max_disconnect_poll,
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback,
drop_cache_rather_verify=self.config.drop_cache_rather_verify,
username=self.config.username,
password=self.config.password,
realm=self.config.realm,
**options)
def init(test=False):
global _db, _testing
if _db and not test:
return
log.info("Initializing zodb")
handle(BeforeDatabaseInitalizedEvent())
if not test:
storage_type = get_config().get('db', 'storage_type')
if storage_type == 'zeo':
from ZODB import DB
storage = ClientStorage('%s/socket' % get_db_dir())
_db = DB(storage)
elif storage_type == 'embedded':
from ZODB import DB
storage = FileStorage('%s/data.fs' % get_db_dir())
_db = DB(storage)
elif storage_type == 'memory':
from ZODB.tests.util import DB
_db = DB()
else:
raise Exception("Unknown storage type '%s'" % storage_type)
else:
from ZODB.tests.util import DB
_db = DB()
_testing = True
init_schema()
def __init__(self) -> None:
"""Initialize the GitHub client and load state from db"""
self.CONFIG = load_config()
self.storage = ClientStorage(self.CONFIG["port"])
self.db = DB(self.storage)
self._client = github3.login(token=self.CONFIG["token"])
self._init_db()
def open_db(options):
if options.db_filename:
storage = FileStorage(options.db_filename, read_only=options.readonly)
else:
storage = ClientStorage(options.zeo_address,
storage=options.zeo_storage,
read_only=options.readonly)
return DB(storage)
def open():
addr = options["ZODB", "zeo_addr"]
if addr and addr[0] == "(" and addr[-1] == ")":
s, p = tuple(addr[1:-1].split(',', 1))
addr = s, int(p)
cs = ClientStorage(addr)
db = ZODB.DB(cs, cache_size=options["ZODB", "cache_size"])
return db
def setSystem(system):
addr = 'localhost', 9001
storage = ClientStorage(addr)
db = DB(storage)
connection = db.open()
root = connection.root()
root["system"] = system
transaction.commit()
db.close()
def _main(host,
port,
unix=None,
days=1,
username=None,
password=None,
realm=None,
blob_dir=None,
storage='1',
shared_blob_dir=True):
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
addr = host, int(port)
if blob_dir:
blob_dir = os.path.abspath(blob_dir)
cs = None
logger = logging.getLogger(__name__)
try:
# We do not want to wait until a zeoserver is up and running; it
# should already be running, so wait=False
cs = ClientStorage(
addr,
storage=storage,
wait=False,
read_only=True,
username=username,
password=password,
realm=realm,
blob_dir=blob_dir,
shared_blob_dir=shared_blob_dir,
)
for i in range(60):
if cs.is_connected():
break
time.sleep(1)
else:
logger.error("Could not connect to zeoserver. Please make sure it "
"is running.")
cs.close()
sys.exit(1)
try:
# The script should not exit until the packing is done.
# => wait=True
cs.pack(wait=True, days=int(days))
except ClientDisconnected:
logger.error("Disconnected from zeoserver. Please make sure it "
"is still running.")
sys.exit(1)
finally:
if cs is not None:
cs.close()
def getSystem():
addr = 'localhost', 9001
storage = ClientStorage(addr)
db = DB(storage)
connection = db.open()
root = connection.root()
if root.has_key("system"):
return root["system"]
else:
return System()
db.close()
def check_server(addr, storage, write):
t0 = time.time()
if ZEO_VERSION == 2:
# TODO: should do retries w/ exponential backoff.
cs = ClientStorage(addr,
storage=storage,
wait=0,
read_only=(not write))
else:
cs = ClientStorage(addr,
storage=storage,
debug=1,
wait_for_server_on_startup=1)
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
if write:
db = ZODB.DB(cs)
cn = db.open()
root = cn.root()
try:
# We store the data in a special `monitor' dict under the root,
# where other tools may also store such heartbeat and bookkeeping
# type data.
monitor = root.get('monitor')
if monitor is None:
monitor = root['monitor'] = PersistentMapping()
obj = monitor['zeoup'] = monitor.get('zeoup', MinPO(0))
obj.value += 1
transaction.commit()
except ConflictError:
pass
cn.close()
db.close()
else:
data, serial = cs.load("\0\0\0\0\0\0\0\0", "")
cs.close()
t1 = time.time()
print("Elapsed time: %.2f" % (t1 - t0))
def main():
z1 = ClientStorage(('localhost', 2001), wait=1)
z2 = ClientStorage(('localhost', 2002), wait=2)
db1 = ZODB.DB(z1)
db2 = ZODB.DB(z2)
c1 = db1.open()
c2 = db2.open()
r1 = c1.root()
r2 = c2.root()
while 1:
try:
try:
update(r1, r2)
except ConflictError, msg:
print msg
transaction.abort()
c1.sync()
c2.sync()
except (ClientDisconnected, DisconnectedError), err:
print "disconnected", err
time.sleep(2)
def pack1(addr, storage, days, wait):
cs = ClientStorage(addr, storage=storage, wait_for_server_on_startup=wait)
if wait:
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
cs._startup()
else:
connect(cs)
cs.invalidator = None
cs.pack(wait=1, days=days)
cs.close()
def __init__(self, hostname=None, port=None):
import Configuration # Please leave this import here, db.py is imported during installation process
cfg = Configuration.Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage = ClientStorage((hostname, port),
username=cfg.getDBUserName(),
password=cfg.getDBPassword(),
realm=cfg.getDBRealm())
self._db = MaKaCDB(self._storage)
self._conn = {}
def setUp(self):
logger.info("setUp() %s", self.id())
port = get_port()
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(
self.getConfig(), zconf, port)
self._pids = [pid]
self._servers = [adminaddr]
self._conf_path = path
self._storage = ClientStorage(zport,
'1',
cache_size=20000000,
min_disconnect_poll=0.5,
wait=1,
wait_timeout=60)
self._storage.registerDB(DummyDB(), None)
def __init__(self):
# return if connection already set
if hasattr(self, "connection"):
logging.debug(f'DB Object already exists. Return')
return
logging.debug('Creating a new db instance')
# start the server if not started already
start_server()
server_and_port = (SERVER, DB_PORT)
self.storage = ClientStorage(server_and_port)
self.db = DB(self.storage)
self.connection = self.db.open()
self.dbroot = self.connection.root()
self.dbroot = self.connection.root()
def open(self):
from ZEO.ClientStorage import ClientStorage
# config.server is a multikey of socket-connection-address values
# where the value is a socket family, address tuple.
L = [server.address for server in self.config.server]
return ClientStorage(
L,
storage=self.config.storage,
cache_size=self.config.cache_size,
name=self.config.name,
client=self.config.client,
var=self.config.var,
min_disconnect_poll=self.config.min_disconnect_poll,
max_disconnect_poll=self.config.max_disconnect_poll,
wait=self.config.wait,
read_only=self.config.read_only,
read_only_fallback=self.config.read_only_fallback)
def __init__(self, hostname=None, port=None, max_disconnect_poll=30):
# Please leave this import here, db.py is imported during installation process
from indico.core.config import Config
cfg = Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage = ClientStorage((hostname, port),
username=cfg.getDBUserName(),
password=cfg.getDBPassword(),
realm=cfg.getDBRealm(),
max_disconnect_poll=max_disconnect_poll)
self._db = MigratedDB(self._storage)
self._conn = threading.local()
self._conn.conn = None
def main():
if len(sys.argv) not in (3, 4):
sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
sys.argv[0])
sys.exit(2)
hostport = sys.argv[1]
delay = float(sys.argv[2])
if sys.argv[3:]:
name = sys.argv[3]
else:
name = "1"
if "/" in hostport:
address = hostport
else:
if ":" in hostport:
i = hostport.index(":")
host, port = hostport[:i], hostport[i + 1:]
else:
host, port = "", hostport
port = int(port)
address = (host, port)
print "Connecting to %s..." % repr(address)
storage = ClientStorage(address, name)
print "Connected. Now starting a transaction..."
oid = storage.new_oid()
version = ""
revid = ZERO
data = MinPO("timeout.py")
pickled_data = zodb_pickle(data)
t = Transaction()
t.user = "******"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, version, t)
print "Stored. Now voting..."
storage.tpc_vote(t)
print "Voted; now sleeping %s..." % delay
time.sleep(delay)
print "Done."
def connect(server=None, zodb=None):
global conn
close() # close any existing connection
if zodb:
conn = DB(FileStorage(os.path.expanduser(zodb))).open()
else:
if not server:
server = cluster.ZEOSERVER
s = server
p = 12345
if ':' in server:
s, p = server.split(':')
p = int(p)
MB = 1024**2
storage = ClientStorage((s, p), cache_size=16 * MB)
db = DB(storage)
conn = db.open()
root = conn.root()
return root
def get_storage(self):
kwargs = {}
kwargs['storage'] = self.storage
kwargs['wait'] = False
kwargs['read_only'] = True
if self.username:
kwargs['username'] = self.username
if self.password:
kwargs['password'] = self.password
if self.realm:
kwargs['realm'] = self.realm
try:
from ZODB.interfaces import IBlobStorage
if self.blob_dir:
kwargs['blob_dir'] = os.path.abspath(self.blob_dir)
kwargs['shared_blob_dir'] = self.shared_blob_dir
except ImportError:
pass
return ClientStorage(self.addr, **kwargs)
def get_storage(zodb_uri):
uri_parts = urlparse(str(zodb_uri))
print colored("Trying to open {}...".format(zodb_uri), 'green')
if uri_parts.scheme == 'zeo':
if uri_parts.port is None:
print colored("No ZEO port specified. Assuming 9675", 'yellow')
storage = ClientStorage((uri_parts.hostname, uri_parts.port or 9675),
username=uri_parts.username,
password=uri_parts.password,
realm=uri_parts.path[1:])
elif uri_parts.scheme in ('file', None):
storage = FileStorage.FileStorage(uri_parts.path)
else:
raise Exception("URI scheme not known: {}".format(uri_parts.scheme))
print colored("Done!", 'green')
return storage
def checkZEOInvalidation(self):
addr = self._storage._addr
storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
try:
oid = self._storage.new_oid()
ob = MinPO('first')
revid1 = self._dostore(oid, data=ob)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('first'))
self.assertEqual(serial, revid1)
revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
for n in range(3):
# Let the server and client talk for a moment.
# Is there a better way to do this?
asyncore.poll(0.1)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('second'),
'Invalidation message was not sent!')
self.assertEqual(serial, revid2)
finally:
storage2.close()
def setUp(self):
logger.info("setUp() %s", self.id())
port = get_port()
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(
self.getConfig(), zconf, port)
self._pids = [pid]
self._servers = [adminaddr]
self._conf_path = path
if not self.blob_cache_dir:
# This is the blob cache for ClientStorage
self.blob_cache_dir = tempfile.mkdtemp()
self._storage = ClientStorage(zport,
'1',
cache_size=20000000,
min_disconnect_poll=0.5,
wait=1,
wait_timeout=60,
blob_dir=self.blob_cache_dir,
shared_blob_dir=self.shared_blob_dir)
self._storage.registerDB(DummyDB())
def checkConnectionInvalidationOnReconnect(self):
storage = ClientStorage(self.addr, wait=1, min_disconnect_poll=0.1)
self._storage = storage
# and we'll wait for the storage to be reconnected:
for i in range(100):
if storage.is_connected():
break
time.sleep(0.1)
else:
raise AssertionError("Couldn't connect to server")
class DummyDB:
_invalidatedCache = 0
def invalidateCache(self):
self._invalidatedCache += 1
def invalidate(*a, **k):
pass
db = DummyDB()
storage.registerDB(db)
base = db._invalidatedCache
# Now we'll force a disconnection and reconnection
storage._connection.close()
# and we'll wait for the storage to be reconnected:
for i in range(100):
if storage.is_connected():
break
time.sleep(0.1)
else:
raise AssertionError("Couldn't connect to server")
# Now, the root object in the connection should have been invalidated:
self.assertEqual(db._invalidatedCache, base + 1)
def initConnection(self, clear=True):
if not (self.conn and self.conn.opened):
if self.url.endswith('.fs'):
from ZODB.FileStorage import FileStorage
if clear and os.path.exists(self.path):
os.unlink('/tmp/zodb_local2.fs')
os.unlink('/tmp/zodb_local2.fs.index')
os.unlink('/tmp/zodb_local2.fs.tmp')
os.unlink('/tmp/zodb_local2.fs.lock')
openstr = os.path.abspath(os.path.expanduser(self.url[7:]))
fs = FileStorage(openstr)
else:
from ZEO.ClientStorage import ClientStorage
schema, opts = _parse_rfc1738_args(self.url)
fs = ClientStorage((opts['host'], int(opts['port'])))
self.zdb = ZODB.DB(fs)
self.conn = self.zdb.open()
root = self.conn.root()
if 'rdflib' not in root:
root['rdflib'] = ConjunctiveGraph(self.store_name)
self.graph = self.g = root['rdflib']
transaction.commit()
def zlibfactory():
# Wraps uri in :class:`zc.slibstorage.ZlibStorage` and returns a
# :class:`ZODB.DB`
# Delay setting the client name until the very end so whatever is going to
# set environment variables will have done so.
if 'client' not in storage_kw:
name = os.environ.get("DATASERVER_ZEO_CLIENT_NAME")
if name: # pragma: no cover This isn't documented.
# storage name is automatically part of it
storage_kw['client'] = name
# ClientCache docs say 200MB is good
storage_kw.setdefault('cache_size', 200 * 1024 * 1024)
# Client storage is very picky: a Unix path must be bytes, not
# unicode
client = ClientStorage(*args, **storage_kw)
if 'demostorage' in orig_kw: # pragma: no cover
client = DemoStorage(base=client)
zlib = ZlibStorage(client)
return DB(zlib, **dbkw)
