def testTrim(self):
am = ActivityMonitor(history_length=0.1)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
time.sleep(0.2)
c._transferred(3, 7)
am.closedConnection(c)
self.assert_(len(am.log) <= 1)
def testAddLogEntries(self):
am = ActivityMonitor(history_length=3600)
self.assertEqual(len(am.log), 0)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
c._transferred(3, 7)
am.closedConnection(c)
self.assertEqual(len(am.log), 2)
def testSetHistoryLength(self):
am = ActivityMonitor(history_length=3600)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
time.sleep(0.2)
c._transferred(3, 7)
am.closedConnection(c)
self.assertEqual(len(am.log), 2)
am.setHistoryLength(0.1)
self.assertEqual(am.getHistoryLength(), 0.1)
self.assertTrue(len(am.log) <= 1)
def _zodbactivity(self, db, suffix):
now = time()
start = now - 15 # Prometheus polls every 15 seconds
end = now
zodb = db._getDB()
if zodb.getActivityMonitor() is None:
zodb.setActivityMonitor(ActivityMonitor())
data = zodb.getActivityMonitor().getActivityAnalysis(start=start,
end=end,
divisions=1)[0]
return [
metric(
"zodb_connections" + suffix,
data["connections"],
"gauge",
"ZODB connections",
self.labels(),
),
metric(
"zodb_load_count" + suffix,
data["loads"],
"counter",
"ZODB load count",
self.labels(),
),
metric(
"zodb_store_count" + suffix,
data["stores"],
"counter",
"ZODB store count",
self.labels(),
),
]
def open(self, database_name, databases):
DB = self.createDB(database_name, databases)
if self.config.connection_class:
# set the connection class
DB.klass = self.config.connection_class
if self.config.class_factory is not None:
DB.classFactory = self.config.class_factory
from ZODB.ActivityMonitor import ActivityMonitor
DB.setActivityMonitor(ActivityMonitor())
return DB
def testSetHistoryLength(self):
am = ActivityMonitor(history_length=3600)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
time.sleep(0.2)
c._transferred(3, 7)
am.closedConnection(c)
self.assertEqual(len(am.log), 2)
am.setHistoryLength(0.1)
self.assertEqual(am.getHistoryLength(), 0.1)
self.assert_(len(am.log) <= 1)
def testTrim(self):
am = ActivityMonitor(history_length=0.1)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
time.sleep(0.2)
c._transferred(3, 7)
am.closedConnection(c)
self.assertTrue(len(am.log) <= 1)
def testAddLogEntries(self):
am = ActivityMonitor(history_length=3600)
self.assertEqual(len(am.log), 0)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
c._transferred(3, 7)
am.closedConnection(c)
self.assertEqual(len(am.log), 2)
def openDB(self, options):
if self.db is not None:
return self.db
db_configuration = options.config.database
db = db_configuration.open()
connection = db.open()
root = connection.root()
app = root.get(ZopePublication.root_name)
if app is None or not ISchoolToolApplication.providedBy(app):
connection.close()
return None
connection.close()
provideUtility(db, IDatabase)
db.setActivityMonitor(ActivityMonitor())
self.db = db
return self.db
def _zodbactivity(self, db, suffix):
now = time()
start = now - 15 # Prometheus polls every 15 seconds
end = now
zodb = db._getDB()
if zodb.getActivityMonitor() is None:
zodb.setActivityMonitor(ActivityMonitor())
data = zodb.getActivityMonitor().getActivityAnalysis(start=start,
end=end,
divisions=1)[0]
return [
metric('zodb_connections' + suffix, data['connections'], 'gauge',
'ZODB connections'),
metric('zodb_load_count' + suffix, data['loads'], 'gauge',
'ZODB load count'),
metric('zodb_store_count' + suffix, data['stores'], 'gauge',
'ZODB store count'),
]
def includeme(config, db_from_uri=db_from_uri):
"""
This includeme recognizes a ``zodbconn.uri`` setting in your deployment
settings and creates a ZODB database if it finds one. ``zodbconn.uri``
is the database URI or URIs (either a whitespace-delimited string, a
carriage-return-delimed string or a list of strings).
Database is activated with `ZODB.ActivityMonitor.ActivityMonitor`.
It will also recognize *named* database URIs as long as an unnamed
database is in the configuration too:
zodbconn.uri.sessions = file:///home/project/var/Data.fs
"""
# db_from_uri in
databases = config.registry._zodb_databases = {}
for name, uri in get_uris(config.registry.settings):
db = db_from_uri(uri, name,
databases) # side effect: populate "databases"
db.setActivityMonitor(ActivityMonitor())
def includeme(config, db_from_uri=db_from_uri, open=open):
"""
This includeme recognizes a ``zodbconn.uri`` setting in your deployment
settings and creates a ZODB database if it finds one. ``zodbconn.uri``
is the database URI or URIs (either a whitespace-delimited string, a
carriage-return-delimed string or a list of strings).
Database is activated with `ZODB.ActivityMonitor.ActivityMonitor`.
It will also recognize *named* database URIs as long as an unnamed
database is in the configuration too:
zodbconn.uri.sessions = file:///home/project/var/Data.fs
Use the key ``zodbconn.transferlog`` in the deployment settings to specify
a filename to write ZODB load/store information to, or leave key's value
blank to send to stdout.
"""
databases = config.registry._zodb_databases = {}
for name, uri in get_uris(config.registry.settings):
db = db_from_uri(uri, name, databases)
# ^^ side effect: populate "databases"
db.setActivityMonitor(ActivityMonitor())
txlog_filename = config.registry.settings.get('zodbconn.transferlog')
if txlog_filename is not None:
if txlog_filename.strip() == '':
stream = sys.stdout
else:
stream = open(txlog_filename, 'a')
txlog_threshhold = config.registry.settings.get(
'zodbconn.transferlog_threshhold')
if txlog_threshhold is not None:
txlog_threshhold = int(txlog_threshhold)
transferlog = TransferLog(stream, txlog_threshhold)
config.add_subscriber(transferlog.start, ZODBConnectionOpened)
config.add_subscriber(transferlog.end, ZODBConnectionWillClose)
config.registry._transferlog = transferlog # for testing only
def testActivityAnalysis(self):
am = ActivityMonitor(history_length=3600)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
c._transferred(3, 7)
am.closedConnection(c)
res = am.getActivityAnalysis(start=0, end=0, divisions=10)
lastend = 0
for n in range(9):
div = res[n]
self.assertEqual(div['stores'], 0)
self.assertEqual(div['loads'], 0)
self.assertTrue(div['start'] > 0)
self.assertTrue(div['start'] >= lastend)
self.assertTrue(div['start'] < div['end'])
lastend = div['end']
div = res[9]
self.assertEqual(div['stores'], 9)
self.assertEqual(div['loads'], 4)
self.assertTrue(div['start'] > 0)
self.assertTrue(div['start'] >= lastend)
self.assertTrue(div['start'] < div['end'])
def testActivityAnalysis(self):
am = ActivityMonitor(history_length=3600)
c = FakeConnection()
c._transferred(1, 2)
am.closedConnection(c)
c._transferred(3, 7)
am.closedConnection(c)
res = am.getActivityAnalysis(start=0, end=0, divisions=10)
lastend = 0
for n in range(9):
div = res[n]
self.assertEqual(div['stores'], 0)
self.assertEqual(div['loads'], 0)
self.assert_(div['start'] > 0)
self.assert_(div['start'] >= lastend)
self.assert_(div['start'] < div['end'])
lastend = div['end']
div = res[9]
self.assertEqual(div['stores'], 9)
self.assertEqual(div['loads'], 4)
self.assert_(div['start'] > 0)
self.assert_(div['start'] >= lastend)
self.assert_(div['start'] < div['end'])
if options.restore_manager:
connection = db.open()
root = connection.root()
app = root[ZopePublication.root_name]
self.restoreManagerUser(app, options.manager_password)
transaction.commit()
connection.close()
# TODO: don't start application when running from command line
#if options.pack or options.restore_manager:
# return db
self.startApplication(db)
provideUtility(db, IDatabase)
db.setActivityMonitor(ActivityMonitor())
return db
def main(self, argv=sys.argv):
options = self.load_options(argv)
db = self.setup(options)
def main():
SchoolToolServer().main()
if __name__ == '__main__':
main()
def startup():
from App.PersistentExtra import patchPersistent
import Globals # to set / fetch data
patchPersistent()
from Zope2.App import patches
patches.apply_patches()
global app
# Import products
OFS.Application.import_products()
configuration = getConfiguration()
# Open the database
dbtab = configuration.dbtab
try:
# Try to use custom storage
try:
m = imp.find_module('custom_zodb', [configuration.testinghome])
except:
m = imp.find_module('custom_zodb', [configuration.instancehome])
except Exception:
# if there is no custom_zodb, use the config file specified databases
DB = dbtab.getDatabase('/', is_root=1)
else:
m = imp.load_module('Zope2.custom_zodb', m[0], m[1], m[2])
sys.modules['Zope2.custom_zodb'] = m
# Get the database and join it to the dbtab multidatabase
# FIXME: this uses internal datastructures of dbtab
databases = getattr(dbtab, 'databases', {})
if hasattr(m, 'DB'):
DB = m.DB
databases.update(getattr(DB, 'databases', {}))
DB.databases = databases
else:
DB = ZODB.DB(m.Storage, databases=databases)
notify(DatabaseOpened(DB))
Globals.BobobaseName = DB.getName()
if DB.getActivityMonitor() is None:
from ZODB.ActivityMonitor import ActivityMonitor
DB.setActivityMonitor(ActivityMonitor())
Globals.DB = DB
Zope2.DB = DB
# Hook for providing multiple transaction object manager undo support:
Globals.UndoManager = DB
Globals.opened.append(DB)
import ClassFactory
DB.classFactory = ClassFactory.ClassFactory
# "Log on" as system user
newSecurityManager(None, AccessControl.User.system)
# Set up the CA
load_zcml()
# Set up the "app" object that automagically opens
# connections
app = App.ZApplication.ZApplicationWrapper(DB, 'Application',
OFS.Application.Application, ())
Zope2.bobo_application = app
# Initialize the app object
application = app()
OFS.Application.initialize(application)
if Globals.DevelopmentMode:
# Set up auto-refresh.
from App.RefreshFuncs import setupAutoRefresh
setupAutoRefresh(application._p_jar)
application._p_jar.close()
# "Log off" as system user
noSecurityManager()
global startup_time
startup_time = asctime()
notify(DatabaseOpenedWithRoot(DB))
Zope2.zpublisher_transactions_manager = TransactionsManager()
Zope2.zpublisher_exception_hook = zpublisher_exception_hook
Zope2.zpublisher_validated_hook = validated_hook
Zope2.__bobo_before__ = noSecurityManager
def __init__(self, zodb_config):
self.databases = {}
for name, uri in zodb_config.items():
db = self.db_from_uri(uri, name, self.databases)
db.setActivityMonitor(ActivityMonitor())
def startup():
import Globals # to set / fetch data
patch_persistent()
global app
# Import products
OFS.Application.import_products()
configuration = getConfiguration()
# Open the database
dbtab = configuration.dbtab
try:
# Try to use custom storage
try:
m = imp.find_module('custom_zodb', [configuration.testinghome])
except:
m = imp.find_module('custom_zodb', [configuration.instancehome])
except Exception:
# if there is no custom_zodb, use the config file specified databases
DB = dbtab.getDatabase('/', is_root=1)
else:
m = imp.load_module('Zope2.custom_zodb', m[0], m[1], m[2])
sys.modules['Zope2.custom_zodb'] = m
# Get the database and join it to the dbtab multidatabase
# FIXME: this uses internal datastructures of dbtab
databases = getattr(dbtab, 'databases', {})
if hasattr(m, 'DB'):
DB = m.DB
databases.update(getattr(DB, 'databases', {}))
DB.databases = databases
else:
DB = ZODB.DB(m.Storage, databases=databases)
# Force a connection to every configured database, to ensure all of them
# can indeed be opened. This avoids surprises during runtime when traversal
# to some database mountpoint fails as the underlying storage cannot be
# opened at all
if dbtab is not None:
for mount, name in dbtab.listMountPaths():
_db = dbtab.getDatabase(mount)
_conn = _db.open()
_conn.close()
del _conn
del _db
notify(DatabaseOpened(DB))
Globals.BobobaseName = DB.getName()
if DB.getActivityMonitor() is None:
from ZODB.ActivityMonitor import ActivityMonitor
DB.setActivityMonitor(ActivityMonitor())
Globals.DB = DB
Zope2.DB = DB
# Hook for providing multiple transaction object manager undo support:
Globals.UndoManager = DB
Globals.opened.append(DB)
import ClassFactory
DB.classFactory = ClassFactory.ClassFactory
# "Log on" as system user
newSecurityManager(None, AccessControl.User.system)
# Set up the CA
load_zcml()
# Set up the "app" object that automagically opens
# connections
app = App.ZApplication.ZApplicationWrapper(DB, 'Application',
OFS.Application.Application, ())
Zope2.bobo_application = app
# Initialize the app object
application = app()
OFS.Application.initialize(application)
application._p_jar.close()
# "Log off" as system user
noSecurityManager()
global startup_time
startup_time = asctime()
notify(DatabaseOpenedWithRoot(DB))
Zope2.zpublisher_transactions_manager = TransactionsManager()
Zope2.zpublisher_exception_hook = zpublisher_exception_hook
Zope2.zpublisher_validated_hook = validated_hook
Zope2.__bobo_before__ = noSecurityManager
