def run_server(cls) -> None:
"""Run the GMB server
Spins up the ZEO database server and a background scheduler to update state
at the configured interval. Throws an exception if the server is already running.
"""
if os.path.exists(CONFIG["pid_file"]):
with open(CONFIG["pid_file"], "r") as fi:
pid = int(fi.read().strip())
if psutil.pid_exists(pid):
raise Exception("Server already running!")
with open(CONFIG["pid_file"], "w") as pidfile:
pidfile.write(str(os.getpid()))
logging.info("Starting server...")
config = load_config()
ZEO.server(path=CONFIG["db_location"], port=config["port"])
client = cls()
sched = BackgroundScheduler(daemon=True)
sched.add_job(
client.update,
"interval",
seconds=config["update_interval"],
next_run_time=datetime.datetime.now(),
)
sched.start()
logging.info("server running")
try:
while True:
time.sleep(2)
except (KeyboardInterrupt, SystemExit):
sched.shutdown()
def admin_db_factory():
return ZEO.DB(
addr,
ssl=ZEO.tests.testssl.client_ssl() if root_cert else nobody_ssl(),
credentials=dict(name='root', password=root_pwd)
if root_password else None,
wait_timeout=19999,
)
def _decide_database_address():
"""When "Testing" is an environment variable, an in-memory server is set up that will be gone after closing the process.
Returns the address of that server, or the address as specified in the `db_config.zeoconf` file."""
if "TESTING" in os.environ:
address, stop = ZEO.server(
) # <- in-memory server that's gone after closing of process.
return address
else:
return ('127.0.0.1', 8091
) # <- As specified in the `db_config.zeoconf` file.
def test_client_side(self):
# First, traditional:
path = tempfile.mkdtemp(prefix='zeo-test-')
self.addCleanup(shutil.rmtree, path)
addr, stop = ZEO.server(os.path.join(path, 'data.fs'), threaded=False)
db = ZEO.DB(addr)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close()
stop()
# Now, do conflict resolution on the client.
addr2, stop = ZEO.server(
storage_conf='<mappingstorage>\n</mappingstorage>\n',
zeo_conf=dict(client_conflict_resolution=True),
threaded=False,
)
db = ZEO.DB(addr2)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
self.assertEqual(conn2.root.l.value, 1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close()
stop()
def database_connection():
"""Sets up a database connection if it has not been set up yet in this process and returns said connection."""
global _database_connection
global _current_process_id
if not _database_connection or os.getpid() != _current_process_id:
_database_connection = ZEO.connection(_database_address)
_current_process_id = os.getpid()
transaction.begin()
# print(f"PID {os.getpid()}: Created new DB connection: {_database_connection} connecting to {_database_address}")
# else:
# print(f"PID {os.getpid()}: Returning old DB connection {_database_connection}")
return _database_connection
def conn(self, request):
"""
Fixture that provides a secondary connection to the same ZEO
"""
tm = transaction.TransactionManager()
db = ZEO.DB(self.zeo.sockpath())
conn = db.open(tm)
app = conn.root.Application
yield helpers.Namespace({'tm': tm, 'app': app})
tm.abort()
conn.close()
def test_client_side(self):
# First, traditional:
addr, stop = ZEO.server('data.fs', threaded=False)
db = ZEO.DB(addr)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close()
stop()
# Now, do conflict resolution on the client.
addr2, stop = ZEO.server(
storage_conf='<mappingstorage>\n</mappingstorage>\n',
zeo_conf=dict(client_conflict_resolution=True),
threaded=False,
)
db = ZEO.DB(addr2)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
self.assertEqual(conn2.root.l.value, 1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close()
stop()
def test_client_side(self):
# First, traditional:
path = tempfile.mkdtemp(prefix='zeo-test-')
self.addCleanup(shutil.rmtree, path)
addr, stop = ZEO.server(os.path.join(path, 'data.fs'), threaded=False)
db = ZEO.DB(addr)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close(); stop()
# Now, do conflict resolution on the client.
addr2, stop = ZEO.server(
storage_conf='<mappingstorage>\n</mappingstorage>\n',
zeo_conf=dict(client_conflict_resolution=True),
threaded=False,
)
db = ZEO.DB(addr2)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
self.assertEqual(conn2.root.l.value, 1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close(); stop()
def test_client_side(self):
# First, traditional:
addr, stop = ZEO.server('data.fs', threaded=False)
db = ZEO.DB(addr)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close(); stop()
# Now, do conflict resolution on the client.
addr2, stop = ZEO.server(
storage_conf='<mappingstorage>\n</mappingstorage>\n',
zeo_conf=dict(client_conflict_resolution=True),
threaded=False,
)
db = ZEO.DB(addr2)
with db.transaction() as conn:
conn.root.l = Length(0)
conn2 = db.open()
conn2.root.l.change(1)
with db.transaction() as conn:
conn.root.l.change(1)
self.assertEqual(conn2.root.l.value, 1)
conn2.transaction_manager.commit()
self.assertEqual(conn2.root.l.value, 2)
db.close(); stop()
def connectZEO(host='docker-compose_zeo_1', port=8090):
'''
Input:
host - host server
port - port to connect to
Output:
returns database-root
'''
# To connect to ZEO
import ZEO
# Connect to ZODB
connection = ZEO.connection((host, port))
return connection.root()
def test_userstats(tempdir):
path = os.path.join(tempdir, 'data.fs')
addr, stop = zerodb.server(path=path)
admin_db = ZEO.DB(addr, ssl=ZEO.tests.testssl.client_ssl())
with admin_db.transaction() as conn:
# The get_admin function gets us an admin object with CRUD methods.
admin = zerodb.permissions.base.get_admin(conn)
admin.add_user('user0', pem_data=(pem_data('cert0')))
admin.add_user('user1', pem_data=(pem_data('cert1')))
admin_db.close()
# Now, let's try connecting
def user_db_factory(n='0'):
return zerodb.DB(
addr,
username='******' + n,
key=user_key,
cert_file=pem_path('cert' + n),
key_file=pem_path('key' + n),
server_cert=ZEO.tests.testssl.server_cert,
)
for u in '01':
db = user_db_factory(u)
for j in range(3):
db._root[j] = db._root.__class__()
db._root[j]['x'] = 'x' * (500 * int(u) + 1)
db._connection.transaction_manager.commit()
db._db.close()
stats = userstats(path)
assert sorted(stats) == [
(b'\x00\x00\x00\x00\x00\x00\x00\x02', 'root', 23759),
(b'\x00\x00\x00\x00\x00\x00\x00\x08', 'user0', 1120),
(b'\x00\x00\x00\x00\x00\x00\x00\t', 'user1', 1154),
]
stop()
def shoot():
"""zodbshootout-inspired performance exercise.
"""
options = parser.parse_args()
concurrency = options.concurrency
object_size = options.object_size
transaction_size = options.transaction_size
repetitions = options.repetitions
if options.ssl:
from ZEO.tests.testssl import server_config, client_ssl
else:
server_config = None
client_ssl = lambda: None
if options.demo_storage_baseline:
db_factory = None
headings = ('add', 'update', 'read')
stop = lambda: None
else:
if options.address:
addr = options.address
if ':' in addr:
host, port = addr.split(':')
addr = host, int(port)
else:
addr = '127.0.0.1', int(addr)
stop = lambda: None
else:
if os.path.exists('perf.fs'):
os.remove('perf.fs')
try:
addr, stop = ZEO.server(threaded=False,
path='perf.fs',
zeo_conf=server_config)
except TypeError:
# ZEO 4
addr, stop = ZEO.server()
db_factory = lambda: ZEO.DB(addr,
ssl=client_ssl(),
wait_timeout=9999,
server_sync=options.server_sync)
if options.read_only:
headings = ('read', 'prefetch')
else:
headings = ('add', 'update', 'cached', 'read', 'prefetch')
# Initialize database
db = db_factory()
with db.transaction() as conn:
conn.root.speedtest = speedtest = BTree()
for ic in range(concurrency):
speedtest[ic] = data = BTree()
for ir in range(repetitions):
data[ir] = P()
db.pack()
db.close()
print('Times per operation in microseconds (o=%s, t=%s, c=%s)' %
(object_size, transaction_size, concurrency))
print(' %12s' * 5 % ('op', 'min', 'mean', 'median', 'max'))
queues = [(multiprocessing.Queue(), multiprocessing.Queue())
for ip in range(concurrency)]
if options.save:
save_file = open(options.save, 'a')
else:
save_file = None
if concurrency > 1 or save_file:
processes = [
multiprocessing.Process(
target=run_test,
args=(db_factory, ip, queues[ip][0], queues[ip][1],
object_size, transaction_size, repetitions,
options.read_only),
) for ip in range(concurrency)
]
for p in processes:
p.daemon = True
p.start()
for iqueue, oqueue in queues:
oqueue.get(timeout=9) # ready?
for name in headings:
for iqueue, oqueue in queues:
iqueue.put(None)
data = [
oqueue.get(timeout=999) / repetitions for _, oqueue in queues
]
summarize(name, data)
if save_file:
save_file.write('\t'.join(
map(str, (options.name, object_size, transaction_size,
concurrency * options.client_hosts, repetitions,
options.server_sync, options.ssl,
options.demo_storage_baseline, options.address,
name, sum(data) / len(data)))) + '\n')
for p in processes:
p.join(1)
else:
[(iqueue, oqueue)] = queues
for name in headings:
iqueue.put(None)
if options.profile:
import cProfile
profiler = cProfile.Profile()
profiler.enable()
else:
profiler = None
run_test(db_factory, 0, iqueue, oqueue, object_size, transaction_size,
repetitions, options.read_only)
oqueue.get(timeout=9) # ready?
if profiler is not None:
profiler.disable()
profiler.dump_stats(options.profile)
for name in headings:
summarize(name, [oqueue.get(timeout=999) / repetitions])
stop()
def run(self) -> None:
ZEO.server(storage_conf=self._storage_conf, zeo_conf=zeo_conf)
asyncore.loop()
import ZEO address, stop = ZEO.server(port="8090")
import ZEO adr, stop = ZEO.server(path='127.0.0.1', port=8018) print(adr) connection = ZEO.connection(adr) client_storage = ZEO.client(adr) stop()
import ZODB.config
import ZEO
import transaction
print('jj')
ZEO.server(path='localhost', port=8018)
db = ZODB.config.databaseFromURL('zodb.config')
connection = db.open()
root = connection.root()
print(root)
root['ab'] = 12
transaction.commit()
#client=ZEO.client(addr)
#connection = ZEO.connection(address)
#db = ZEO.DB(address)
#connection = db.open()
def shoot():
"""zodbshootout-inspired performance exercise.
"""
options = parser.parse_args()
concurrency = options.concurrency
object_size = options.object_size
transaction_size = options.transaction_size
repetitions = options.repetitions
if options.ssl:
from ZEO.tests.testssl import server_config, client_ssl
else:
server_config = None
client_ssl = lambda : None
if options.demo_storage_baseline:
db_factory = None
headings = ('add', 'update', 'read')
stop = lambda : None
else:
if options.address:
addr = options.address
if ':' in addr:
host, port = addr.split(':')
addr = host, int(port)
else:
addr = '127.0.0.1', int(addr)
stop = lambda : None
else:
if os.path.exists('perf.fs'):
os.remove('perf.fs')
try:
addr, stop = ZEO.server(
threaded=False, path='perf.fs', zeo_conf=server_config)
except TypeError:
# ZEO 4
addr, stop = ZEO.server()
db_factory = lambda : ZEO.DB(
addr,
ssl=client_ssl(),
wait_timeout=9999,
server_sync=options.server_sync)
if options.read_only:
headings = ('read', 'prefetch')
else:
headings = ('add', 'update', 'cached', 'read', 'prefetch')
# Initialize database
db = db_factory()
with db.transaction() as conn:
conn.root.speedtest = speedtest = BTree()
for ic in range(concurrency):
speedtest[ic] = data = BTree()
for ir in range(repetitions):
data[ir] = P()
db.pack()
db.close()
print('Times per operation in microseconds (o=%s, t=%s, c=%s)' % (
object_size, transaction_size, concurrency))
print(' %12s' * 5 % ('op', 'min', 'mean', 'median', 'max'))
queues = [(multiprocessing.Queue(), multiprocessing.Queue())
for ip in range(concurrency)]
if options.save:
save_file = open(options.save, 'a')
else:
save_file = None
if concurrency > 1 or save_file:
processes = [
multiprocessing.Process(
target=run_test,
args=(db_factory, ip, queues[ip][0], queues[ip][1],
object_size, transaction_size, repetitions,
options.read_only),
)
for ip in range(concurrency)
]
for p in processes:
p.daemon = True
p.start()
for iqueue, oqueue in queues:
oqueue.get(timeout=9) # ready?
for name in headings:
for iqueue, oqueue in queues:
iqueue.put(None)
data = [oqueue.get(timeout=999) / repetitions
for _, oqueue in queues]
summarize(name, data)
if save_file:
save_file.write('\t'.join(
map(str,
(options.name,
object_size, transaction_size,
concurrency * options.client_hosts, repetitions,
options.server_sync, options.ssl,
options.demo_storage_baseline,
options.address,
name, sum(data)/len(data))
)
) + '\n')
for p in processes:
p.join(1)
else:
[(iqueue, oqueue)] = queues
for name in headings:
iqueue.put(None)
if options.profile:
import cProfile
profiler = cProfile.Profile()
profiler.enable()
else:
profiler = None
run_test(db_factory, 0, iqueue, oqueue,
object_size, transaction_size, repetitions,
options.read_only)
oqueue.get(timeout=9) # ready?
if profiler is not None:
profiler.disable()
profiler.dump_stats(options.profile)
for name in headings:
summarize(name, [oqueue.get(timeout=999) / repetitions])
stop()
