def test_zip_import_component_from_config(self): sio = StringIO(''' <schema> <abstracttype name="something"/> <section name="*" attribute="something" type="something" /> </schema> ''') schema = ZConfig.loadSchemaFile(sio) value = ''' %import foo.sample <sample> data value </sample> ''' sio = StringIO(value) config, _ = ZConfig.loadConfigFile(schema, sio) self.assertEqual(config.something.data, "| value |") sio = StringIO(value) with self.assertRaises(ZConfig.ConfigurationSyntaxError): ZConfig.loadConfigFile(schema, sio, overrides=["sample/data=othervalue"])
def checkConfigureViaZConfig(self): replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf') if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' conf = u""" %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <mysql> driver auto db %s user relstoragetest passwd relstoragetest </mysql> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dbname, ) schema_xml = u""" <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from io import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = db.storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.mysql import MySQLAdapter self.assertIsInstance(adapter, MySQLAdapter) self.assertEqual(adapter._params, { 'passwd': 'relstoragetest', 'db': dbname, 'user': '******', }) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual( adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close()
def run_with_options(options): conf_fn = options.config_file # Do the gevent stuff ASAP if getattr(options, 'gevent', False): import gevent.monkey gevent.monkey.patch_all() if options.log: import logging lvl_map = getattr(logging, '_nameToLevel', None) or getattr(logging, '_levelNames', {}) logging.basicConfig(level=lvl_map.get(options.log, logging.INFO), format='%(asctime)s %(levelname)-5.5s [%(name)s][%(thread)d:%(process)d][%(threadName)s] %(message)s') object_size = max(options.object_size, pobject_base_size) if options.profile_dir and not os.path.exists(options.profile_dir): os.makedirs(options.profile_dir) schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _handler = ZConfig.loadConfigFile(schema, conf_fn) contenders = [(db.name, db) for db in config.databases] if options.zap: _zap(contenders) # results: {contender_name: {concurrency_level: {objects_per_txn: [[SpeedTestTimes]...]}}} results = defaultdict(lambda: defaultdict(dict)) try: for objects_per_txn in options.counts or DEFAULT_OBJ_COUNTS: for concurrency in options.concurrency or DEFAULT_CONCURRENCIES: speedtest = SpeedTest( concurrency, objects_per_txn, object_size, options.profile_dir, 'threads' if options.threads else 'mp', test_reps=options.test_reps) if options.btrees: import BTrees if options.btrees == 'IO': speedtest.MappingType = BTrees.family64.IO.BTree else: speedtest.MappingType = BTrees.family64.OO.BTree for contender_name, db in contenders: print(( 'Testing %s with objects_per_txn=%d, object_size=%d, ' 'mappingtype=%s and concurrency=%d (threads? %s)' % (contender_name, objects_per_txn, object_size, speedtest.MappingType, concurrency, options.threads)), file=sys.stderr) all_times = _run_one_contender(options, speedtest, contender_name, db) #results[key] = all_times results[contender_name][concurrency][objects_per_txn] = all_times # The finally clause causes test results to print even if the tests # stop early. finally: _print_results(options, contenders, results)
def open_storages(options): schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfigFile(schema, options.config_file) source = config.source.open() destination = config.destination.open() return source, destination
def setup_eventlog(self): """Create an eventlog ZConfig configuration and patch it onto the global config, so it's present when ftw.structlog attempts to read it to derive its own logfile path from the eventlog's logfile path. """ schema = ZConfig.loadSchemaFile( StringIO(""" <schema> <import package='ZConfig.components.logger'/> <section type='eventlog' name='*' attribute='eventlog'/> </schema> """)) fn = self.mktemp() eventlog_conf, handler = ZConfig.loadConfigFile( schema, StringIO(""" <eventlog> <logfile> path %s level debug </logfile> </eventlog> """ % fn)) assert eventlog_conf.eventlog is not None getConfiguration().eventlog = eventlog_conf.eventlog
def databaseFromFile(f): """Create a database from a file object that provides configuration. See :func:`databaseFromString`. """ config, handle = ZConfig.loadConfigFile(getDbSchema(), f) return databaseFromConfig(config.database)
def checkConfigureViaZConfig(self): replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf') if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' dsn = ( "dbname='%s' user='******' password='******'" % dbname) conf = """ %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <postgresql> dsn %s </postgresql> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dsn, ) schema_xml = """ <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from StringIO import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, handler = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = getattr(db, 'storage', None) if storage is None: # ZODB < 3.9 storage = db._storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.postgresql import PostgreSQLAdapter self.assert_(isinstance(adapter, PostgreSQLAdapter)) self.assertEqual(adapter._dsn, dsn) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual( adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close()
def main(args=None): optparser = argparse.ArgumentParser( description="Script to check validity of a configuration file", epilog=""" Each file named on the command line is checked for syntactical errors and schema conformance. The schema must be specified. If no files are specified and standard input is not a TTY, standard in is treated as a configuration file. Specifying a schema and no configuration files causes the schema to be checked.""", ) optparser.add_argument("-s", "--schema", dest="schema", required=True, help="use the schema in FILE (can be a URL)", metavar="FILE") optparser.add_argument( "file", nargs='*', help="Optional configuration file to check", type=argparse.FileType('r'), ) options = optparser.parse_args(args=args) schema = ZConfig.loadSchema(options.schema) if not options.file: if sys.stdin.isatty(): # just checking the schema return 0 # stdin is a pipe options.file = [sys.stdin] errors = False for f in options.file: try: ZConfig.loadConfigFile(schema, f) except ZConfig.ConfigurationError as e: print(str(e), file=sys.stderr) errors = True return int(errors)
def checkConfigureViaZConfig(self): replica_conf = os.path.join(os.path.dirname(relstorage.tests.__file__), 'replicas.conf') dsn = 'dbname=' + self.dbname conf = u""" %%import relstorage %%import newt.db <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <newt> <postgresql> driver auto dsn %s </postgresql> </newt> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dsn, ) schema_xml = u""" <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from io import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = db.storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.postgresql import PostgreSQLAdapter self.assertIsInstance(adapter, PostgreSQLAdapter) self.assertEqual(adapter._dsn, dsn) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual(adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) from .._adapter import Adapter self.assertEqual(Adapter, storage._adapter.__class__) finally: db.close()
def loadConfiguration(file, url=None): global _schema if _schema is None: here = os.path.dirname(os.path.abspath(__file__)) path = os.path.join(here, "schema", "productconfig.xml") _schema = ZConfig.loadSchema(path) data, handlers = ZConfig.loadConfigFile(_schema, file, url=url) return dict((sect.getSectionName(), sect.mapping) for sect in data.product_config)
def checkConfigureViaZConfig(self): replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf') if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' dsn = ("dbname='%s' user='******' password='******'" % dbname) conf = """ %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <postgresql> dsn %s </postgresql> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dsn, ) schema_xml = """ <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from StringIO import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, handler = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = getattr(db, 'storage', None) if storage is None: # ZODB < 3.9 storage = db._storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.postgresql import PostgreSQLAdapter self.assert_(isinstance(adapter, PostgreSQLAdapter)) self.assertEqual(adapter._dsn, dsn) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual(adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close()
def testGoodConfigLoad(self): """A bunch of correct configuration files""" global _good_configs for config in _good_configs: good_config = StringIO(config) conf, handler = ZConfig.loadConfigFile(self.schema, good_config) datatypes._paths = [] return
def configure(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. schema = self.schema sio = cStringIO.StringIO(text.replace("<<INSTANCE_HOME>>", TEMPNAME)) conf, handler = ZConfig.loadConfigFile(schema, sio) self.assertEqual(conf.instancehome, TEMPNAME) setConfiguration(conf)
def main(argv=None): if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-d", "--days", dest="days", default=1, help="Days of history to keep (default: %(default)s)", type=float, ) parser.add_argument( "--prepack", dest="prepack", default=False, action="store_true", help="Perform only the pre-pack preparation stage of a pack. " "(Only works with some storage types)", ) parser.add_argument( "--use-prepack-state", dest="reuse_prepack", default=False, action="store_true", help="Skip the preparation stage and go straight to packing. " "Requires that a pre-pack has been run, or that packing was aborted " "before it was completed.", ) parser.add_argument("config_file", type=argparse.FileType('r')) options = parser.parse_args(argv[1:]) logging.basicConfig( level=logging.DEBUG, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfigFile(schema, options.config_file) t = time.time() - options.days * 86400.0 for s in config.storages: name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__) logger.info("Opening %s...", name) storage = s.open() logger.info("Packing %s.", name) if options.prepack or options.reuse_prepack: storage.pack(t, ZODB.serialize.referencesf, prepack_only=options.prepack, skip_prepack=options.reuse_prepack) else: # Be non-relstorage Storages friendly storage.pack(t, ZODB.serialize.referencesf) storage.close() logger.info("Packed %s.", name)
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. text = text.replace("<<INSTANCE_HOME>>", self.TEMPNAME) sio = io.StringIO(text) conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio) self.assertEqual(conf.instancehome, self.TEMPNAME) return conf
def config(configfile, schemafile=None, features=()): # Load the configuration schema if schemafile is None: schemafile = os.path.join(os.path.dirname(appsetup.__file__), 'schema', 'schema.xml') # Let's support both, an opened file and path if isinstance(schemafile, basestring): schema = ZConfig.loadSchema(schemafile) else: schema = ZConfig.loadSchemaFile(schemafile) # Load the configuration file # Let's support both, an opened file and path try: if isinstance(configfile, basestring): options, handlers = ZConfig.loadConfig(schema, configfile) else: options, handlers = ZConfig.loadConfigFile(schema, configfile) except ZConfig.ConfigurationError as msg: sys.stderr.write("Error: %s\n" % str(msg)) sys.exit(2) # Insert all specified Python paths if options.path: sys.path[:0] = [os.path.abspath(p) for p in options.path] # Parse product configs zope.app.appsetup.product.setProductConfigurations(options.product_config) # Setup the event log options.eventlog() # Setup other defined loggers for logger in options.loggers: logger() # Insert the devmode feature, if turned on if options.devmode: features += ('devmode', ) logging.warning( "Developer mode is enabled: this is a security risk " "and should NOT be enabled on production servers. Developer mode " "can usually be turned off by setting the `devmode` option to " "`off` or by removing it from the instance configuration file " "completely.") # Execute the ZCML configuration. appsetup.config(options.site_definition, features=features) # Connect to and open the database, notify subscribers. db = appsetup.multi_database(options.databases)[0][0] notify(zope.processlifetime.DatabaseOpened(db)) return db
def config(configfile, schemafile=None, features=()): # Load the configuration schema if schemafile is None: schemafile = os.path.join( os.path.dirname(appsetup.__file__), 'schema', 'schema.xml') # Let's support both, an opened file and path if isinstance(schemafile, basestring): schema = ZConfig.loadSchema(schemafile) else: schema = ZConfig.loadSchemaFile(schemafile) # Load the configuration file # Let's support both, an opened file and path try: if isinstance(configfile, basestring): options, handlers = ZConfig.loadConfig(schema, configfile) else: options, handlers = ZConfig.loadConfigFile(schema, configfile) except ZConfig.ConfigurationError as msg: sys.stderr.write("Error: %s\n" % str(msg)) sys.exit(2) # Insert all specified Python paths if options.path: sys.path[:0] = [os.path.abspath(p) for p in options.path] # Parse product configs zope.app.appsetup.product.setProductConfigurations( options.product_config) # Setup the event log options.eventlog() # Setup other defined loggers for logger in options.loggers: logger() # Insert the devmode feature, if turned on if options.devmode: features += ('devmode',) logging.warning("Developer mode is enabled: this is a security risk " "and should NOT be enabled on production servers. Developer mode " "can usually be turned off by setting the `devmode` option to " "`off` or by removing it from the instance configuration file " "completely.") # Execute the ZCML configuration. appsetup.config(options.site_definition, features=features) # Connect to and open the database, notify subscribers. db = appsetup.multi_database(options.databases)[0][0] notify(zope.processlifetime.DatabaseOpened(db)) return db
class ZopeStarterTestCase(LoggingTestHelper, unittest.TestCase): schema = None def setUp(self): if self.schema is None: ZopeStarterTestCase.schema = getSchema() LoggingTestHelper.setUp(self) def tearDown(self): try: os.rmdir(TEMPPRODUCTS) os.rmdir(TEMPNAME) except: pass Products.__path__ = [d for d in Products.__path__ if os.path.exists(d)] LoggingTestHelper.tearDown(self) # reset logger states for name in (None, 'access', 'trace'): logger = logging.getLogger(name) logger.__dict__.update(logger_states[name]) def _clearHandlers(self): from ZConfig.components.logger import loghandler del loghandler._reopenable_handlers[:] def get_starter(self, conf): starter = Zope2.Startup.get_starter() starter.setConfiguration(conf) return starter def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. schema = self.schema sio = cStringIO.StringIO( text.replace("<<INSTANCE_HOME>>", TEMPNAME)) try: os.mkdir(TEMPNAME) os.mkdir(TEMPPRODUCTS) except OSError, why: if why == 17: # already exists pass conf, self.handler = ZConfig.loadConfigFile(schema, sio) self.assertEqual(conf.instancehome, TEMPNAME) return conf
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. sio = io.BytesIO(text.replace("<<INSTANCE_HOME>>", self.TEMPNAME)) try: os.mkdir(self.TEMPNAME) except OSError as why: if why == 17: # already exists pass conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio) self.assertEqual(conf.instancehome, self.TEMPNAME) return conf
def configure(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. schema = self.schema text = text.replace("<<INSTANCE_HOME>>", TEMPNAME) text = text.replace("<<PRODUCTS>>", TEMPPRODUCTS) text = text.replace("<<PRODUCTS2>>", TEMPPRODUCTS2) sio = cStringIO.StringIO(text) conf, handler = ZConfig.loadConfigFile(schema, sio) from Zope2.Startup.handlers import handleConfig handleConfig(conf, handler) self.assertEqual(conf.instancehome, TEMPNAME) setConfiguration(conf)
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. sio = io.BytesIO(text.replace("<<INSTANCE_HOME>>", TEMPNAME)) os.mkdir(TEMPNAME) os.mkdir(TEMPVAR) try: conf, handler = ZConfig.loadConfigFile(getSchema(), sio) finally: os.rmdir(TEMPVAR) os.rmdir(TEMPNAME) self.assertEqual(conf.instancehome, TEMPNAME) return conf, handler
def test_zeo_config(self): # We're looking for a port that doesn't exist so a # connection attempt will fail. Instead of elaborate # logic to loop over a port calculation, we'll just pick a # simple "random", likely to not-exist port number and add # an elaborate comment explaining this instead. Go ahead, # grep for 9. from ZEO.ClientStorage import ClientDisconnected import ZConfig from ZODB.config import getDbSchema from StringIO import StringIO cfg = """ <zodb> <zeoclient> server localhost:56897 wait false </zeoclient> </zodb> """ config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg)) self.assertEqual(config.database.config.storage.config.blob_dir, None) self.assertRaises(ClientDisconnected, self._test, cfg) cfg = """ <zodb> <zeoclient> blob-dir /tmp server localhost:56897 wait false </zeoclient> </zodb> """ config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg)) self.assertEqual(config.database.config.storage.config.blob_dir, '/tmp') self.assertRaises(ClientDisconnected, self._test, cfg)
def test_zeo_config(self): # We're looking for a port that doesn't exist so a # connection attempt will fail. Instead of elaborate # logic to loop over a port calculation, we'll just pick a # simple "random", likely to not-exist port number and add # an elaborate comment explaining this instead. Go ahead, # grep for 9. from ZEO.ClientStorage import ClientDisconnected import ZConfig from ZODB.config import getDbSchema from StringIO import StringIO cfg = """ <zodb> <zeoclient> server localhost:56897 wait false </zeoclient> </zodb> """ config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg)) self.assertEqual(config.database[0].config.storage.config.blob_dir, None) self.assertRaises(ClientDisconnected, self._test, cfg) cfg = """ <zodb> <zeoclient> blob-dir blobs server localhost:56897 wait false </zeoclient> </zodb> """ config, handle = ZConfig.loadConfigFile(getDbSchema(), StringIO(cfg)) self.assertEqual(config.database[0].config.storage.config.blob_dir, 'blobs') self.assertRaises(ClientDisconnected, self._test, cfg)
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. schema = self.schema sio = cStringIO.StringIO( text.replace("<<INSTANCE_HOME>>", TEMPNAME)) os.mkdir(TEMPNAME) os.mkdir(TEMPPRODUCTS) try: conf, handler = ZConfig.loadConfigFile(schema, sio) finally: os.rmdir(TEMPPRODUCTS) os.rmdir(TEMPNAME) self.assertEqual(conf.instancehome, TEMPNAME) return conf, handler
def test_zip_import_component_from_config(self): sio = StringIO(''' <schema> <abstracttype name="something"/> <section name="*" attribute="something" type="something" /> </schema> ''') schema = ZConfig.loadSchemaFile(sio) sio = StringIO(''' %import foo.sample <sample> data value </sample> ''') config, _ = ZConfig.loadConfigFile(schema, sio) self.assertEqual(config.something.data, "| value |")
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. text = text.replace("<<INSTANCE_HOME>>", TEMPNAME) if six.PY2: sio = io.BytesIO(text) else: sio = io.StringIO(text) os.mkdir(TEMPNAME) os.mkdir(TEMPVAR) try: conf, handler = ZConfig.loadConfigFile(getSchema(), sio) finally: os.rmdir(TEMPVAR) os.rmdir(TEMPNAME) self.assertEqual(conf.instancehome, TEMPNAME) return conf, handler
def load_config_text(self, text): # We have to create a directory of our own since the existence # of the directory is checked. This handles this in a # platform-independent way. text = text.replace("<<INSTANCE_HOME>>", self.TEMPNAME) if six.PY2: sio = io.BytesIO(text) else: sio = io.StringIO(text) try: os.mkdir(self.TEMPNAME) except OSError as why: if why == 17: # already exists pass conf, self.handler = ZConfig.loadConfigFile(getSchema(), sio) self.assertEqual(conf.instancehome, self.TEMPNAME) return conf
def config(configfile, schemafile=None, features=()): # Load the configuration schema if schemafile is None: schemafile = os.path.join( os.path.dirname(appsetup.__file__), 'schema', 'schema.xml') # Let's support both, an opened file and path if isinstance(schemafile, basestring): schema = ZConfig.loadSchema(schemafile) else: schema = ZConfig.loadSchemaFile(schemafile) # Load the configuration file # Let's support both, an opened file and path try: if isinstance(configfile, basestring): options, handlers = ZConfig.loadConfig(schema, configfile) else: options, handlers = ZConfig.loadConfigFile(schema, configfile) except ZConfig.ConfigurationError, msg: sys.stderr.write("Error: %s\n" % str(msg)) sys.exit(2)
def setup_eventlog(self): schema = ZConfig.loadSchemaFile( StringIO(""" <schema> <import package='ZConfig.components.logger'/> <section type='eventlog' name='*' attribute='eventlog'/> </schema> """)) self.tempdir = tempfile.mkdtemp() f = open(os.path.join(self.tempdir, 'instance0.log'), 'w') f.close() eventlog_conf = ZConfig.loadConfigFile( schema, StringIO(""" <eventlog> <logfile> path {} level debug </logfile> </eventlog> """.format(f.name)))[0] assert eventlog_conf.eventlog is not None getConfiguration().eventlog = eventlog_conf.eventlog
def storageFromFile(f): config, handle = ZConfig.loadConfigFile(getStorageSchema(), f) return storageFromConfig(config.storage)
def run_with_options(options): conf_fn = options.config_file # Do the gevent stuff ASAP if getattr(options, 'gevent', False): # Because of what we import up top, this must have # already been done, to be sure that it's effective import gevent.monkey if not gevent.monkey.is_module_patched('threading'): raise AssertionError("gevent monkey-patching should have happened") if options.log: import logging lvl_map = getattr(logging, '_nameToLevel', None) or getattr(logging, '_levelNames', {}) logging.basicConfig(level=lvl_map.get(options.log, logging.INFO), format='%(asctime)s %(levelname)-5.5s [%(name)s][%(thread)d:%(process)d][%(threadName)s] %(message)s') object_size = max(options.object_size, pobject_base_size) if options.profile_dir and not os.path.exists(options.profile_dir): os.makedirs(options.profile_dir) schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _handler = ZConfig.loadConfigFile(schema, conf_fn) contenders = [(db.name, db) for db in config.databases] if options.zap: _zap(contenders) # results: {contender_name: {concurrency_level: {objects_per_txn: [[SpeedTestTimes]...]}}} results = defaultdict(lambda: defaultdict(dict)) try: for objects_per_txn in options.counts or DEFAULT_OBJ_COUNTS: for concurrency in options.concurrency or DEFAULT_CONCURRENCIES: speedtest = SpeedTest( concurrency, objects_per_txn, object_size, options.profile_dir, mp_strategy=(options.threads or 'mp'), test_reps=options.test_reps, use_blobs=options.use_blobs) speedtest.min_object_count = options.min_object_count if options.btrees: import BTrees if options.btrees == 'IO': speedtest.MappingType = BTrees.family64.IO.BTree else: speedtest.MappingType = BTrees.family64.OO.BTree for contender_name, db in contenders: print(( 'Testing %s with objects_per_txn=%d, object_size=%d, ' 'mappingtype=%s, objecttype=%s, min_objects=%d and concurrency=%d (threads? %s)' % (contender_name, objects_per_txn, object_size, speedtest.MappingType, speedtest.ObjectType, options.min_object_count, concurrency, options.threads)), file=sys.stderr) all_times = _run_one_contender(options, speedtest, contender_name, db) #results[key] = all_times results[contender_name][concurrency][objects_per_txn] = all_times # The finally clause causes test results to print even if the tests # stop early. finally: _print_results(options, contenders, results)
def storageFromFile(f): """Create a storage from a file object providing storage-configuration. """ config, handle = ZConfig.loadConfigFile(getStorageSchema(), f) return storageFromConfig(config.storage)
def databaseFromFile(f): config, handle = ZConfig.loadConfigFile(getDbSchema(), f) return databaseFromConfig(config.database)
def checkConfigureViaZConfig(self): replica_conf = os.path.join(os.path.dirname(__file__), 'replicas.conf') if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' conf = u""" %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB cache-local-dir-read-count 12 cache-local-dir-write-max-size 10MB <mysql> driver auto db %s user relstoragetest passwd relstoragetest </mysql> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dbname, ) schema_xml = u""" <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from io import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = db.storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.mysql import MySQLAdapter self.assertIsInstance(adapter, MySQLAdapter) self.assertEqual( adapter._params, { 'passwd': 'relstoragetest', 'db': dbname, 'user': '******', }) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual(adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close()
def get_config(self, text): conf, handler = ZConfig.loadConfigFile(self.get_schema(), ZConfig._compat.NStringIO(text)) self.assertTrue(not handler) return conf
def get_config(self, text): conf, handler = ZConfig.loadConfigFile(self.get_schema(), StringIO.StringIO(text)) self.assert_(not handler) return conf
def load_factory(self, text): conf, xxx = ZConfig.loadConfigFile(self.get_schema(), StringIO.StringIO(text)) self.assertEqual(len(conf.servers), 1) return conf.servers[0]
def setUp(self): super(ConfigObjectTest, self).setUp() self.zconf, handler = ZConfig.loadConfigFile(self.schema, StringIO(GOOD_CONFIG2)) datatypes._paths = [] return
def checkConfigureViaZConfig(self): # pylint:disable=too-many-locals import tempfile dsn = os.environ.get('ORACLE_TEST_DSN', 'XE') fd, replica_conf = tempfile.mkstemp() os.write(fd, dsn.encode("ascii")) os.close(fd) try: if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' conf = u""" %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <oracle> user %s password relstoragetest dsn %s </oracle> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dbname, dsn, ) schema_xml = u""" <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from io import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _handler = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = db.storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.oracle import OracleAdapter self.assertIsInstance(adapter, OracleAdapter) self.assertEqual(adapter._user, dbname) self.assertEqual(adapter._password, 'relstoragetest') self.assertEqual(adapter._dsn, dsn) self.assertEqual(adapter._twophase, False) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual( adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close() finally: os.remove(replica_conf)
def loadfile(self, file_or_path): schema = self.get_schema() self.conf, self.handlers = ZConfig.loadConfigFile(schema, file_or_path) return self.conf
def loadfile(self, file): schema = self.get_schema() self.conf, self.handlers = ZConfig.loadConfigFile(schema, file) return self.conf
def load_configfile(self): sio = StringIO(self.__configuration) cfg = ZConfig.loadConfigFile(self.schema, sio, self.zconfig_options) self.configroot, self.confighandlers = cfg
def checkConfigureViaZConfig(self): import tempfile dsn = os.environ.get('ORACLE_TEST_DSN', 'XE') fd, replica_conf = tempfile.mkstemp() os.write(fd, dsn) os.close(fd) try: if self.keep_history: dbname = base_dbname else: dbname = base_dbname + '_hf' conf = """ %%import relstorage <zodb main> <relstorage> name xyz read-only false keep-history %s replica-conf %s blob-chunk-size 10MB <oracle> user %s password relstoragetest dsn %s </oracle> </relstorage> </zodb> """ % ( self.keep_history and 'true' or 'false', replica_conf, dbname, dsn, ) schema_xml = """ <schema> <import package="ZODB"/> <section type="ZODB.database" name="main" attribute="database"/> </schema> """ import ZConfig from StringIO import StringIO schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, handler = ZConfig.loadConfigFile(schema, StringIO(conf)) db = config.database.open() try: storage = getattr(db, 'storage', None) if storage is None: # ZODB < 3.9 storage = db._storage self.assertEqual(storage.isReadOnly(), False) self.assertEqual(storage.getName(), "xyz") adapter = storage._adapter from relstorage.adapters.oracle import OracleAdapter self.assert_(isinstance(adapter, OracleAdapter)) self.assertEqual(adapter._user, dbname) self.assertEqual(adapter._password, 'relstoragetest') self.assertEqual(adapter._dsn, dsn) self.assertEqual(adapter._twophase, False) self.assertEqual(adapter.keep_history, self.keep_history) self.assertEqual( adapter.connmanager.replica_selector.replica_conf, replica_conf) self.assertEqual(storage._options.blob_chunk_size, 10485760) finally: db.close() finally: os.remove(replica_conf)