def test_zip_import_component_from_config(self):
sio = StringIO('''
<schema>
<abstracttype name="something"/>
<section name="*"
attribute="something"
type="something"
/>
</schema>
''')
schema = ZConfig.loadSchemaFile(sio)
value = '''
%import foo.sample
<sample>
data value
</sample>
'''
sio = StringIO(value)
config, _ = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(config.something.data, "| value |")
sio = StringIO(value)
with self.assertRaises(ZConfig.ConfigurationSyntaxError):
ZConfig.loadConfigFile(schema, sio,
overrides=["sample/data=othervalue"])
def test_zip_import_component_from_config(self):
sio = StringIO('''
<schema>
<abstracttype name="something"/>
<section name="*"
attribute="something"
type="something"
/>
</schema>
''')
schema = ZConfig.loadSchemaFile(sio)
value = '''
%import foo.sample
<sample>
data value
</sample>
'''
sio = StringIO(value)
config, _ = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(config.something.data, "| value |")
sio = StringIO(value)
with self.assertRaises(ZConfig.ConfigurationSyntaxError):
ZConfig.loadConfigFile(schema, sio,
overrides=["sample/data=othervalue"])
def checkConfigureViaZConfig(self):
replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf')
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
conf = u"""
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<mysql>
driver auto
db %s
user relstoragetest
passwd relstoragetest
</mysql>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dbname,
)
schema_xml = u"""
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from io import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = db.storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.mysql import MySQLAdapter
self.assertIsInstance(adapter, MySQLAdapter)
self.assertEqual(adapter._params, {
'passwd': 'relstoragetest',
'db': dbname,
'user': '******',
})
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(
adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
def run_with_options(options):
conf_fn = options.config_file
# Do the gevent stuff ASAP
if getattr(options, 'gevent', False):
import gevent.monkey
gevent.monkey.patch_all()
if options.log:
import logging
lvl_map = getattr(logging, '_nameToLevel', None) or getattr(logging, '_levelNames', {})
logging.basicConfig(level=lvl_map.get(options.log, logging.INFO),
format='%(asctime)s %(levelname)-5.5s [%(name)s][%(thread)d:%(process)d][%(threadName)s] %(message)s')
object_size = max(options.object_size, pobject_base_size)
if options.profile_dir and not os.path.exists(options.profile_dir):
os.makedirs(options.profile_dir)
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _handler = ZConfig.loadConfigFile(schema, conf_fn)
contenders = [(db.name, db) for db in config.databases]
if options.zap:
_zap(contenders)
# results: {contender_name: {concurrency_level: {objects_per_txn: [[SpeedTestTimes]...]}}}
results = defaultdict(lambda: defaultdict(dict))
try:
for objects_per_txn in options.counts or DEFAULT_OBJ_COUNTS:
for concurrency in options.concurrency or DEFAULT_CONCURRENCIES:
speedtest = SpeedTest(
concurrency, objects_per_txn,
object_size,
options.profile_dir,
'threads' if options.threads else 'mp',
test_reps=options.test_reps)
if options.btrees:
import BTrees
if options.btrees == 'IO':
speedtest.MappingType = BTrees.family64.IO.BTree
else:
speedtest.MappingType = BTrees.family64.OO.BTree
for contender_name, db in contenders:
print((
'Testing %s with objects_per_txn=%d, object_size=%d, '
'mappingtype=%s and concurrency=%d (threads? %s)'
% (contender_name, objects_per_txn, object_size,
speedtest.MappingType,
concurrency, options.threads)), file=sys.stderr)
all_times = _run_one_contender(options, speedtest, contender_name, db)
#results[key] = all_times
results[contender_name][concurrency][objects_per_txn] = all_times
# The finally clause causes test results to print even if the tests
# stop early.
finally:
_print_results(options, contenders, results)
def open_storages(options):
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfigFile(schema, options.config_file)
source = config.source.open()
destination = config.destination.open()
return source, destination
def setup_eventlog(self):
"""Create an eventlog ZConfig configuration and patch it onto the
global config, so it's present when ftw.structlog attempts to read it
to derive its own logfile path from the eventlog's logfile path.
"""
schema = ZConfig.loadSchemaFile(
StringIO("""
<schema>
<import package='ZConfig.components.logger'/>
<section type='eventlog' name='*' attribute='eventlog'/>
</schema>
"""))
fn = self.mktemp()
eventlog_conf, handler = ZConfig.loadConfigFile(
schema,
StringIO("""
<eventlog>
<logfile>
path %s
level debug
</logfile>
</eventlog>
""" % fn))
assert eventlog_conf.eventlog is not None
getConfiguration().eventlog = eventlog_conf.eventlog
def databaseFromFile(f):
"""Create a database from a file object that provides configuration.
See :func:`databaseFromString`.
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
return databaseFromConfig(config.database)
def checkConfigureViaZConfig(self):
replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf')
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
dsn = (
"dbname='%s' user='******' password='******'"
% dbname)
conf = """
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<postgresql>
dsn %s
</postgresql>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dsn,
)
schema_xml = """
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from StringIO import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = getattr(db, 'storage', None)
if storage is None:
# ZODB < 3.9
storage = db._storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.postgresql import PostgreSQLAdapter
self.assert_(isinstance(adapter, PostgreSQLAdapter))
self.assertEqual(adapter._dsn, dsn)
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(
adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
def main(args=None):
optparser = argparse.ArgumentParser(
description="Script to check validity of a configuration file",
epilog="""
Each file named on the command line is checked for syntactical errors
and schema conformance. The schema must be specified. If no files
are specified and standard input is not a TTY, standard in is treated
as a configuration file. Specifying a schema and no configuration
files causes the schema to be checked.""",
)
optparser.add_argument("-s",
"--schema",
dest="schema",
required=True,
help="use the schema in FILE (can be a URL)",
metavar="FILE")
optparser.add_argument(
"file",
nargs='*',
help="Optional configuration file to check",
type=argparse.FileType('r'),
)
options = optparser.parse_args(args=args)
schema = ZConfig.loadSchema(options.schema)
if not options.file:
if sys.stdin.isatty():
# just checking the schema
return 0
# stdin is a pipe
options.file = [sys.stdin]
errors = False
for f in options.file:
try:
ZConfig.loadConfigFile(schema, f)
except ZConfig.ConfigurationError as e:
print(str(e), file=sys.stderr)
errors = True
return int(errors)
def checkConfigureViaZConfig(self):
replica_conf = os.path.join(os.path.dirname(relstorage.tests.__file__),
'replicas.conf')
dsn = 'dbname=' + self.dbname
conf = u"""
%%import relstorage
%%import newt.db
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<newt>
<postgresql>
driver auto
dsn %s
</postgresql>
</newt>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dsn,
)
schema_xml = u"""
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from io import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = db.storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.postgresql import PostgreSQLAdapter
self.assertIsInstance(adapter, PostgreSQLAdapter)
self.assertEqual(adapter._dsn, dsn)
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
from .._adapter import Adapter
self.assertEqual(Adapter, storage._adapter.__class__)
finally:
db.close()
def loadConfiguration(file, url=None):
global _schema
if _schema is None:
here = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(here, "schema", "productconfig.xml")
_schema = ZConfig.loadSchema(path)
data, handlers = ZConfig.loadConfigFile(_schema, file, url=url)
return dict((sect.getSectionName(), sect.mapping)
for sect in data.product_config)
def checkConfigureViaZConfig(self):
replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf')
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
dsn = ("dbname='%s' user='******' password='******'" %
dbname)
conf = """
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<postgresql>
dsn %s
</postgresql>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dsn,
)
schema_xml = """
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from StringIO import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = getattr(db, 'storage', None)
if storage is None:
# ZODB < 3.9
storage = db._storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.postgresql import PostgreSQLAdapter
self.assert_(isinstance(adapter, PostgreSQLAdapter))
self.assertEqual(adapter._dsn, dsn)
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
def testGoodConfigLoad(self):
"""A bunch of correct configuration files"""
global _good_configs
for config in _good_configs:
good_config = StringIO(config)
conf, handler = ZConfig.loadConfigFile(self.schema, good_config)
datatypes._paths = []
return
def configure(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
sio = cStringIO.StringIO(text.replace("<<INSTANCE_HOME>>", TEMPNAME))
conf, handler = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(conf.instancehome, TEMPNAME)
setConfiguration(conf)
def configure(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
sio = cStringIO.StringIO(text.replace("<<INSTANCE_HOME>>", TEMPNAME))
conf, handler = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(conf.instancehome, TEMPNAME)
setConfiguration(conf)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-d",
"--days",
dest="days",
default=1,
help="Days of history to keep (default: %(default)s)",
type=float,
)
parser.add_argument(
"--prepack",
dest="prepack",
default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_argument(
"--use-prepack-state",
dest="reuse_prepack",
default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
parser.add_argument("config_file", type=argparse.FileType('r'))
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfigFile(schema, options.config_file)
t = time.time() - options.days * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
logger.info("Opening %s...", name)
storage = s.open()
logger.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t,
ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
logger.info("Packed %s.", name)
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
text = text.replace("<<INSTANCE_HOME>>", self.TEMPNAME)
sio = io.StringIO(text)
conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio)
self.assertEqual(conf.instancehome, self.TEMPNAME)
return conf
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
text = text.replace("<<INSTANCE_HOME>>", self.TEMPNAME)
sio = io.StringIO(text)
conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio)
self.assertEqual(conf.instancehome, self.TEMPNAME)
return conf
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(os.path.dirname(appsetup.__file__), 'schema',
'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError as msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
# Insert all specified Python paths
if options.path:
sys.path[:0] = [os.path.abspath(p) for p in options.path]
# Parse product configs
zope.app.appsetup.product.setProductConfigurations(options.product_config)
# Setup the event log
options.eventlog()
# Setup other defined loggers
for logger in options.loggers:
logger()
# Insert the devmode feature, if turned on
if options.devmode:
features += ('devmode', )
logging.warning(
"Developer mode is enabled: this is a security risk "
"and should NOT be enabled on production servers. Developer mode "
"can usually be turned off by setting the `devmode` option to "
"`off` or by removing it from the instance configuration file "
"completely.")
# Execute the ZCML configuration.
appsetup.config(options.site_definition, features=features)
# Connect to and open the database, notify subscribers.
db = appsetup.multi_database(options.databases)[0][0]
notify(zope.processlifetime.DatabaseOpened(db))
return db
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(
os.path.dirname(appsetup.__file__), 'schema', 'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError as msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
# Insert all specified Python paths
if options.path:
sys.path[:0] = [os.path.abspath(p) for p in options.path]
# Parse product configs
zope.app.appsetup.product.setProductConfigurations(
options.product_config)
# Setup the event log
options.eventlog()
# Setup other defined loggers
for logger in options.loggers:
logger()
# Insert the devmode feature, if turned on
if options.devmode:
features += ('devmode',)
logging.warning("Developer mode is enabled: this is a security risk "
"and should NOT be enabled on production servers. Developer mode "
"can usually be turned off by setting the `devmode` option to "
"`off` or by removing it from the instance configuration file "
"completely.")
# Execute the ZCML configuration.
appsetup.config(options.site_definition, features=features)
# Connect to and open the database, notify subscribers.
db = appsetup.multi_database(options.databases)[0][0]
notify(zope.processlifetime.DatabaseOpened(db))
return db
class ZopeStarterTestCase(LoggingTestHelper, unittest.TestCase):
schema = None
def setUp(self):
if self.schema is None:
ZopeStarterTestCase.schema = getSchema()
LoggingTestHelper.setUp(self)
def tearDown(self):
try:
os.rmdir(TEMPPRODUCTS)
os.rmdir(TEMPNAME)
except:
pass
Products.__path__ = [d for d in Products.__path__
if os.path.exists(d)]
LoggingTestHelper.tearDown(self)
# reset logger states
for name in (None, 'access', 'trace'):
logger = logging.getLogger(name)
logger.__dict__.update(logger_states[name])
def _clearHandlers(self):
from ZConfig.components.logger import loghandler
del loghandler._reopenable_handlers[:]
def get_starter(self, conf):
starter = Zope2.Startup.get_starter()
starter.setConfiguration(conf)
return starter
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
sio = cStringIO.StringIO(
text.replace("<<INSTANCE_HOME>>", TEMPNAME))
try:
os.mkdir(TEMPNAME)
os.mkdir(TEMPPRODUCTS)
except OSError, why:
if why == 17:
# already exists
pass
conf, self.handler = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(conf.instancehome, TEMPNAME)
return conf
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
sio = io.BytesIO(text.replace("<<INSTANCE_HOME>>", self.TEMPNAME))
try:
os.mkdir(self.TEMPNAME)
except OSError as why:
if why == 17:
# already exists
pass
conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio)
self.assertEqual(conf.instancehome, self.TEMPNAME)
return conf
def configure(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
text = text.replace("<<INSTANCE_HOME>>", TEMPNAME)
text = text.replace("<<PRODUCTS>>", TEMPPRODUCTS)
text = text.replace("<<PRODUCTS2>>", TEMPPRODUCTS2)
sio = cStringIO.StringIO(text)
conf, handler = ZConfig.loadConfigFile(schema, sio)
from Zope2.Startup.handlers import handleConfig
handleConfig(conf, handler)
self.assertEqual(conf.instancehome, TEMPNAME)
setConfiguration(conf)
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
sio = io.BytesIO(text.replace("<<INSTANCE_HOME>>", TEMPNAME))
os.mkdir(TEMPNAME)
os.mkdir(TEMPVAR)
try:
conf, handler = ZConfig.loadConfigFile(getSchema(), sio)
finally:
os.rmdir(TEMPVAR)
os.rmdir(TEMPNAME)
self.assertEqual(conf.instancehome, TEMPNAME)
return conf, handler
def configure(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
text = text.replace("<<INSTANCE_HOME>>", TEMPNAME)
text = text.replace("<<PRODUCTS>>", TEMPPRODUCTS)
text = text.replace("<<PRODUCTS2>>", TEMPPRODUCTS2)
sio = cStringIO.StringIO(text)
conf, handler = ZConfig.loadConfigFile(schema, sio)
from Zope2.Startup.handlers import handleConfig
handleConfig(conf, handler)
self.assertEqual(conf.instancehome, TEMPNAME)
setConfiguration(conf)
def test_zeo_config(self):
# We're looking for a port that doesn't exist so a
# connection attempt will fail. Instead of elaborate
# logic to loop over a port calculation, we'll just pick a
# simple "random", likely to not-exist port number and add
# an elaborate comment explaining this instead. Go ahead,
# grep for 9.
from ZEO.ClientStorage import ClientDisconnected
import ZConfig
from ZODB.config import getDbSchema
from StringIO import StringIO
cfg = """
<zodb>
<zeoclient>
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database.config.storage.config.blob_dir,
None)
self.assertRaises(ClientDisconnected, self._test, cfg)
cfg = """
<zodb>
<zeoclient>
blob-dir /tmp
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database.config.storage.config.blob_dir,
'/tmp')
self.assertRaises(ClientDisconnected, self._test, cfg)
def test_zeo_config(self):
# We're looking for a port that doesn't exist so a
# connection attempt will fail. Instead of elaborate
# logic to loop over a port calculation, we'll just pick a
# simple "random", likely to not-exist port number and add
# an elaborate comment explaining this instead. Go ahead,
# grep for 9.
from ZEO.ClientStorage import ClientDisconnected
import ZConfig
from ZODB.config import getDbSchema
from StringIO import StringIO
cfg = """
<zodb>
<zeoclient>
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
None)
self.assertRaises(ClientDisconnected, self._test, cfg)
cfg = """
<zodb>
<zeoclient>
blob-dir blobs
server localhost:56897
wait false
</zeoclient>
</zodb>
"""
config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg))
self.assertEqual(config.database[0].config.storage.config.blob_dir,
'blobs')
self.assertRaises(ClientDisconnected, self._test, cfg)
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
schema = self.schema
sio = cStringIO.StringIO(
text.replace("<<INSTANCE_HOME>>", TEMPNAME))
os.mkdir(TEMPNAME)
os.mkdir(TEMPPRODUCTS)
try:
conf, handler = ZConfig.loadConfigFile(schema, sio)
finally:
os.rmdir(TEMPPRODUCTS)
os.rmdir(TEMPNAME)
self.assertEqual(conf.instancehome, TEMPNAME)
return conf, handler
def test_zip_import_component_from_config(self):
sio = StringIO('''
<schema>
<abstracttype name="something"/>
<section name="*"
attribute="something"
type="something"
/>
</schema>
''')
schema = ZConfig.loadSchemaFile(sio)
sio = StringIO('''
%import foo.sample
<sample>
data value
</sample>
''')
config, _ = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(config.something.data, "| value |")
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
text = text.replace("<<INSTANCE_HOME>>", TEMPNAME)
if six.PY2:
sio = io.BytesIO(text)
else:
sio = io.StringIO(text)
os.mkdir(TEMPNAME)
os.mkdir(TEMPVAR)
try:
conf, handler = ZConfig.loadConfigFile(getSchema(), sio)
finally:
os.rmdir(TEMPVAR)
os.rmdir(TEMPNAME)
self.assertEqual(conf.instancehome, TEMPNAME)
return conf, handler
def load_config_text(self, text):
# We have to create a directory of our own since the existence
# of the directory is checked. This handles this in a
# platform-independent way.
text = text.replace("<<INSTANCE_HOME>>", self.TEMPNAME)
if six.PY2:
sio = io.BytesIO(text)
else:
sio = io.StringIO(text)
try:
os.mkdir(self.TEMPNAME)
except OSError as why:
if why == 17:
# already exists
pass
conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio)
self.assertEqual(conf.instancehome, self.TEMPNAME)
return conf
def test_zip_import_component_from_config(self):
sio = StringIO('''
<schema>
<abstracttype name="something"/>
<section name="*"
attribute="something"
type="something"
/>
</schema>
''')
schema = ZConfig.loadSchemaFile(sio)
sio = StringIO('''
%import foo.sample
<sample>
data value
</sample>
''')
config, _ = ZConfig.loadConfigFile(schema, sio)
self.assertEqual(config.something.data, "| value |")
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(
os.path.dirname(appsetup.__file__), 'schema', 'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError, msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
def setup_eventlog(self):
schema = ZConfig.loadSchemaFile(
StringIO("""
<schema>
<import package='ZConfig.components.logger'/>
<section type='eventlog' name='*' attribute='eventlog'/>
</schema>
"""))
self.tempdir = tempfile.mkdtemp()
f = open(os.path.join(self.tempdir, 'instance0.log'), 'w')
f.close()
eventlog_conf = ZConfig.loadConfigFile(
schema,
StringIO("""
<eventlog>
<logfile>
path {}
level debug
</logfile>
</eventlog>
""".format(f.name)))[0]
assert eventlog_conf.eventlog is not None
getConfiguration().eventlog = eventlog_conf.eventlog
def storageFromFile(f):
config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
return storageFromConfig(config.storage)
def run_with_options(options):
conf_fn = options.config_file
# Do the gevent stuff ASAP
if getattr(options, 'gevent', False):
# Because of what we import up top, this must have
# already been done, to be sure that it's effective
import gevent.monkey
if not gevent.monkey.is_module_patched('threading'):
raise AssertionError("gevent monkey-patching should have happened")
if options.log:
import logging
lvl_map = getattr(logging, '_nameToLevel', None) or getattr(logging, '_levelNames', {})
logging.basicConfig(level=lvl_map.get(options.log, logging.INFO),
format='%(asctime)s %(levelname)-5.5s [%(name)s][%(thread)d:%(process)d][%(threadName)s] %(message)s')
object_size = max(options.object_size, pobject_base_size)
if options.profile_dir and not os.path.exists(options.profile_dir):
os.makedirs(options.profile_dir)
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _handler = ZConfig.loadConfigFile(schema, conf_fn)
contenders = [(db.name, db) for db in config.databases]
if options.zap:
_zap(contenders)
# results: {contender_name: {concurrency_level: {objects_per_txn: [[SpeedTestTimes]...]}}}
results = defaultdict(lambda: defaultdict(dict))
try:
for objects_per_txn in options.counts or DEFAULT_OBJ_COUNTS:
for concurrency in options.concurrency or DEFAULT_CONCURRENCIES:
speedtest = SpeedTest(
concurrency, objects_per_txn,
object_size,
options.profile_dir,
mp_strategy=(options.threads or 'mp'),
test_reps=options.test_reps,
use_blobs=options.use_blobs)
speedtest.min_object_count = options.min_object_count
if options.btrees:
import BTrees
if options.btrees == 'IO':
speedtest.MappingType = BTrees.family64.IO.BTree
else:
speedtest.MappingType = BTrees.family64.OO.BTree
for contender_name, db in contenders:
print((
'Testing %s with objects_per_txn=%d, object_size=%d, '
'mappingtype=%s, objecttype=%s, min_objects=%d and concurrency=%d (threads? %s)'
% (contender_name, objects_per_txn, object_size,
speedtest.MappingType, speedtest.ObjectType, options.min_object_count,
concurrency, options.threads)), file=sys.stderr)
all_times = _run_one_contender(options, speedtest, contender_name, db)
#results[key] = all_times
results[contender_name][concurrency][objects_per_txn] = all_times
# The finally clause causes test results to print even if the tests
# stop early.
finally:
_print_results(options, contenders, results)
def storageFromFile(f):
"""Create a storage from a file object providing storage-configuration.
"""
config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
return storageFromConfig(config.storage)
def databaseFromFile(f):
config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
return databaseFromConfig(config.database)
def checkConfigureViaZConfig(self):
replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf')
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
conf = u"""
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
cache-local-dir-read-count 12
cache-local-dir-write-max-size 10MB
<mysql>
driver auto
db %s
user relstoragetest
passwd relstoragetest
</mysql>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dbname,
)
schema_xml = u"""
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from io import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = db.storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.mysql import MySQLAdapter
self.assertIsInstance(adapter, MySQLAdapter)
self.assertEqual(
adapter._params, {
'passwd': 'relstoragetest',
'db': dbname,
'user': '******',
})
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
def get_config(self, text):
conf, handler = ZConfig.loadConfigFile(self.get_schema(),
ZConfig._compat.NStringIO(text))
self.assertTrue(not handler)
return conf
def get_config(self, text):
conf, handler = ZConfig.loadConfigFile(self.get_schema(),
StringIO.StringIO(text))
self.assert_(not handler)
return conf
def databaseFromFile(f):
config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
return databaseFromConfig(config.database)
def load_factory(self, text):
conf, xxx = ZConfig.loadConfigFile(self.get_schema(),
StringIO.StringIO(text))
self.assertEqual(len(conf.servers), 1)
return conf.servers[0]
def get_config(self, text):
conf, handler = ZConfig.loadConfigFile(self.get_schema(),
StringIO.StringIO(text))
self.assert_(not handler)
return conf
def setUp(self):
super(ConfigObjectTest, self).setUp()
self.zconf, handler = ZConfig.loadConfigFile(self.schema, StringIO(GOOD_CONFIG2))
datatypes._paths = []
return
def checkConfigureViaZConfig(self):
# pylint:disable=too-many-locals
import tempfile
dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
fd, replica_conf = tempfile.mkstemp()
os.write(fd, dsn.encode("ascii"))
os.close(fd)
try:
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
conf = u"""
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<oracle>
user %s
password relstoragetest
dsn %s
</oracle>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dbname,
dsn,
)
schema_xml = u"""
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from io import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _handler = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = db.storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.oracle import OracleAdapter
self.assertIsInstance(adapter, OracleAdapter)
self.assertEqual(adapter._user, dbname)
self.assertEqual(adapter._password, 'relstoragetest')
self.assertEqual(adapter._dsn, dsn)
self.assertEqual(adapter._twophase, False)
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(
adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
finally:
os.remove(replica_conf)
def storageFromFile(f):
config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
return storageFromConfig(config.storage)
def loadfile(self, file_or_path):
schema = self.get_schema()
self.conf, self.handlers = ZConfig.loadConfigFile(schema, file_or_path)
return self.conf
def loadfile(self, file):
schema = self.get_schema()
self.conf, self.handlers = ZConfig.loadConfigFile(schema, file)
return self.conf
def load_configfile(self):
sio = StringIO(self.__configuration)
cfg = ZConfig.loadConfigFile(self.schema, sio, self.zconfig_options)
self.configroot, self.confighandlers = cfg
def load_configfile(self):
sio = StringIO(self.__configuration)
cfg = ZConfig.loadConfigFile(self.schema, sio, self.zconfig_options)
self.configroot, self.confighandlers = cfg
def checkConfigureViaZConfig(self):
import tempfile
dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
fd, replica_conf = tempfile.mkstemp()
os.write(fd, dsn)
os.close(fd)
try:
if self.keep_history:
dbname = base_dbname
else:
dbname = base_dbname + '_hf'
conf = """
%%import relstorage
<zodb main>
<relstorage>
name xyz
read-only false
keep-history %s
replica-conf %s
blob-chunk-size 10MB
<oracle>
user %s
password relstoragetest
dsn %s
</oracle>
</relstorage>
</zodb>
""" % (
self.keep_history and 'true' or 'false',
replica_conf,
dbname,
dsn,
)
schema_xml = """
<schema>
<import package="ZODB"/>
<section type="ZODB.database" name="main" attribute="database"/>
</schema>
"""
import ZConfig
from StringIO import StringIO
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfigFile(schema, StringIO(conf))
db = config.database.open()
try:
storage = getattr(db, 'storage', None)
if storage is None:
# ZODB < 3.9
storage = db._storage
self.assertEqual(storage.isReadOnly(), False)
self.assertEqual(storage.getName(), "xyz")
adapter = storage._adapter
from relstorage.adapters.oracle import OracleAdapter
self.assert_(isinstance(adapter, OracleAdapter))
self.assertEqual(adapter._user, dbname)
self.assertEqual(adapter._password, 'relstoragetest')
self.assertEqual(adapter._dsn, dsn)
self.assertEqual(adapter._twophase, False)
self.assertEqual(adapter.keep_history, self.keep_history)
self.assertEqual(
adapter.connmanager.replica_selector.replica_conf,
replica_conf)
self.assertEqual(storage._options.blob_chunk_size, 10485760)
finally:
db.close()
finally:
os.remove(replica_conf)
def load_factory(self, text):
conf, xxx = ZConfig.loadConfigFile(self.get_schema(),
StringIO.StringIO(text))
self.assertEqual(len(conf.servers), 1)
return conf.servers[0]
