def runwsgi():
parser = argparse.ArgumentParser()
parser.add_argument('-w', '--webdav', action='store_true')
parser.add_argument('address', help='<ip>:<port>')
parser.add_argument('zope_conf', help='path to zope.conf')
args = parser.parse_args()
startup = os.path.dirname(Zope2.Startup.__file__)
schema = ZConfig.loadSchema(os.path.join(startup, 'zopeschema.xml'))
conf, _ = ZConfig.loadConfig(schema, args.zope_conf)
make_wsgi_app({}, zope_conf=args.zope_conf)
from Signals.SignalHandler import SignalHandler
SignalHandler.registerHandler(signal.SIGTERM, sys.exit)
ip, port = splitport(args.address)
port = int(port)
createServer(
app_wrapper(
large_file_threshold=conf.large_file_threshold,
webdav_ports=[port] if args.webdav else ()),
listen=args.address,
logger=logging.getLogger("access"),
threads=conf.zserver_threads,
).run()
def databaseFromURL(url):
"""Load a database from URL (or file name) that provides configuration.
See :func:`databaseFromString`.
"""
config, handler = ZConfig.loadConfig(getDbSchema(), url)
return databaseFromConfig(config.database)
def test_partition_softupdates(self):
""" Verify that partition sizes are converted correctly """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
for part in config.Partitions.PartitionMap[0].Partition:
if (part.type == 'swap'):
# The swap partition should be 4GB, or 8,388,608 512-byte blocks
self.assertEquals(part.size, 8388608)
def setUp(self):
subs = copy.deepcopy(CONFIG_SUBS)
subs['@INSTALLROOT@'] = INSTALLROOT
os.mkdir(INSTALLROOT)
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
farbconfig, handler = ZConfig.loadConfig(SCHEMA, RELEASE_CONFIG_FILE)
# Copy in each release and package root needed
for release in RELEASE_NAMES:
rewrite_config(
CDROM_INF_IN, CDROM_INF,
{'@CD_VERSION_LINE@': 'CD_VERSION = ' + release.upper()})
releasedest = os.path.join(BUILDROOT, release, 'releaseroot',
builder.RELEASE_CD_PATH)
utils.copyRecursive(ISO_MOUNTPOINT, releasedest)
# The fake CD image we have is for 6.2, so we might need to rename a
# directory
os.rename(os.path.join(releasedest, '6.2-RELEASE'),
os.path.join(releasedest, release.upper()))
# Create the package root and its package directory
pkgdest = os.path.join(BUILDROOT, release, 'pkgroot', 'usr',
'ports', 'packages')
utils.copyRecursive(PACKAGEDIR, pkgdest)
self.nbr = runner.NetInstallAssemblerRunner(farbconfig)
self.nbr.run()
def __call__(self, uri):
(scheme, netloc, path, query, frag) = urlparse.urlsplit(uri)
# urlparse doesnt understand file URLs and stuffs everything into path
(scheme, netloc, path, query, frag) = urlparse.urlsplit("http:" + path)
path = os.path.normpath(path)
schema_xml = self.schema_xml_template
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, path)
for config_item in config.databases + config.storages:
if not frag:
# use the first defined in the file
break
elif frag == config_item.name:
# match found
break
else:
raise KeyError("No storage or database named %s found" % frag)
if isinstance(config_item, ZODBDatabase):
config = config_item.config
factory = config.storage
dbkw = {"connection_cache_size": config.cache_size, "connection_pool_size": config.pool_size}
if config.database_name:
dbkw["database_name"] = config.database_name
else:
factory = config_item
dbkw = dict(cgi.parse_qsl(query))
return factory.open, dbkw
def load(self, relurl, context=None):
url = CONFIG_BASE + relurl
self.conf, self.handlers = ZConfig.loadConfig(self.get_schema(), url)
conf = self.conf
self.assertTrue(conf.getSectionName() is None)
self.assertTrue(conf.getSectionType() is None)
return conf
def __call__(self, uri):
(scheme, netloc, path, query, frag) = urlparse.urlsplit(uri)
# urlparse doesnt understand file URLs and stuffs everything into path
(scheme, netloc, path, query, frag) = urlparse.urlsplit('http:' + path)
path = os.path.normpath(path)
schema_xml = self.schema_xml_template
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, path)
for config_item in config.databases + config.storages:
if not frag:
# use the first defined in the file
break
elif frag == config_item.name:
# match found
break
else:
raise KeyError("No storage or database named %s found" % frag)
if isinstance(config_item, ZODBDatabase):
config = config_item.config
factory = config.storage
dbkw = {
'connection_cache_size': config.cache_size,
'connection_pool_size': config.pool_size,
}
if config.database_name:
dbkw['database_name'] = config.database_name
else:
factory = config_item
dbkw = dict(cgi.parse_qsl(query))
return factory.open, dbkw
def test_partition_softupdates(self):
""" Verify that partition sizes are converted correctly """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
for part in config.Partitions.PartitionMap[0].Partition:
if (part.type == 'swap'):
# The swap partition should be 4GB, or 8,388,608 512-byte blocks
self.assertEquals(part.size, 8388608)
def load(self, relurl, context=None):
url = CONFIG_BASE + relurl
self.conf, self.handlers = ZConfig.loadConfig(self.get_schema(), url)
conf = self.conf
self.assertTrue(conf.getSectionName() is None)
self.assertTrue(conf.getSectionType() is None)
return conf
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear",
dest="clear",
action="store_true",
help="Clear the contents of the destination storage before copying")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' %
(TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination):
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
def test_ports_cvs(self):
""" Test using CVS for ports in binary release """
subs = CONFIG_SUBS.copy()
subs['@PORTSOURCE@'] = 'CVSRoot ' + CVSROOT
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[2]
self.assertEquals(release.cvsroot, CVSROOT)
def test_binary_release(self):
""" Load a binary release configuration """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[2]
# Make sure options were set correctly
self.assertEquals(release.binaryrelease, True)
self.assertEquals(release.useportsnap, True)
self.assertEquals(release.iso, os.path.join(DATA_DIR, 'fake_cd.iso'))
def test_ports_cvs(self):
""" Test using CVS for ports in binary release """
subs = CONFIG_SUBS.copy()
subs['@PORTSOURCE@'] = 'CVSRoot ' + CVSROOT
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[2]
self.assertEquals(release.cvsroot, CVSROOT)
def test_default_dists(self):
""" Ensure that default Dists are set properly """
subs = CONFIG_SUBS.copy()
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[1]
self.assertEquals(release.dists, ["base", "kernels", "doc", "games", "manpages", "catpages", "proflibs", "dict", "info", "src"])
self.assertEquals(release.kerneldists, ["GENERIC", "SMP"])
def test_binary_release(self):
""" Load a binary release configuration """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[2]
# Make sure options were set correctly
self.assertEquals(release.binaryrelease, True)
self.assertEquals(release.useportsnap, True)
self.assertEquals(release.iso, os.path.join(DATA_DIR, 'fake_cd.iso'))
def load(self, relurl, context=None):
url = CONFIG_BASE + relurl
self.conf, self.handlers = ZConfig.loadConfig(self.get_schema(), url)
conf = self.conf
#self.assertEqual(conf.url, url)
self.assert_(conf.getSectionName() is None)
self.assert_(conf.getSectionType() is None)
#self.assert_(conf.delegate is None)
return conf
def load(self, relurl, context=None):
url = CONFIG_BASE + relurl
self.conf, self.handlers = ZConfig.loadConfig(self.get_schema(), url)
conf = self.conf
#self.assertEqual(conf.url, url)
self.assert_(conf.getSectionName() is None)
self.assert_(conf.getSectionType() is None)
#self.assert_(conf.delegate is None)
return conf
def setUp(self):
# Load ZConfig schema
self.schema = ZConfig.loadSchema(farb.CONFIG_SCHEMA)
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, CONFIG_SUBS)
self.config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
self.instSection = self.config.Installations.Installation[0]
self.releaseSection = self.config.Releases.Release[0]
self.instSectionNoCommands = self.config.Installations.Installation[1]
self.instSectionNoDisks = self.config.Installations.Installation[2]
def options(args):
"""Password-specific options loaded from regular ZEO config file."""
try:
opts, args = getopt.getopt(args, "dr:p:f:C:", ["configure=",
"protocol=",
"filename=",
"realm"])
except getopt.error as msg:
usage(msg)
config = None
delete = 0
auth_protocol = None
auth_db = ""
auth_realm = None
for k, v in opts:
if k == '-C' or k == '--configure':
schemafile = os.path.join(os.path.dirname(ZEO.__file__),
"schema.xml")
schema = ZConfig.loadSchema(schemafile)
config, nil = ZConfig.loadConfig(schema, v)
if k == '-d' or k == '--delete':
delete = 1
if k == '-p' or k == '--protocol':
auth_protocol = v
if k == '-f' or k == '--filename':
auth_db = v
if k == '-r' or k == '--realm':
auth_realm = v
if config is not None:
if auth_protocol or auth_db:
usage("Error: Conflicting options; use either -C *or* -p and -f")
auth_protocol = config.zeo.authentication_protocol
auth_db = config.zeo.authentication_database
auth_realm = config.zeo.authentication_realm
elif not (auth_protocol and auth_db):
usage("Error: Must specifiy configuration file or protocol and database")
password = None
if delete:
if not args:
usage("Error: Must specify a username to delete")
elif len(args) > 1:
usage("Error: Too many arguments")
username = args[0]
else:
if not args:
usage("Error: Must specify a username")
elif len(args) > 2:
usage("Error: Too many arguments")
elif len(args) == 1:
username = args[0]
else:
username, password = args
return auth_protocol, auth_db, auth_realm, delete, username, password
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-d",
"--days",
dest="days",
default=0,
help="Days of history to keep (default 0)",
type=float,
)
parser.add_argument(
"--prepack",
dest="prepack",
default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_argument(
"--use-prepack-state",
dest="reuse_prepack",
default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
parser.add_argument("config_file")
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfig(schema, options.config_file)
t = time.time() - options.days * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
log.info("Opening %s...", name)
storage = s.open()
log.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t,
ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
log.info("Packed %s.", name)
def options(args):
"""Password-specific options loaded from regular ZEO config file."""
try:
opts, args = getopt.getopt(
args, "dr:p:f:C:",
["configure=", "protocol=", "filename=", "realm"])
except getopt.error as msg:
usage(msg)
config = None
delete = 0
auth_protocol = None
auth_db = ""
auth_realm = None
for k, v in opts:
if k == '-C' or k == '--configure':
schemafile = os.path.join(os.path.dirname(ZEO.__file__),
"schema.xml")
schema = ZConfig.loadSchema(schemafile)
config, nil = ZConfig.loadConfig(schema, v)
if k == '-d' or k == '--delete':
delete = 1
if k == '-p' or k == '--protocol':
auth_protocol = v
if k == '-f' or k == '--filename':
auth_db = v
if k == '-r' or k == '--realm':
auth_realm = v
if config is not None:
if auth_protocol or auth_db:
usage("Error: Conflicting options; use either -C *or* -p and -f")
auth_protocol = config.zeo.authentication_protocol
auth_db = config.zeo.authentication_database
auth_realm = config.zeo.authentication_realm
elif not (auth_protocol and auth_db):
usage(
"Error: Must specifiy configuration file or protocol and database")
password = None
if delete:
if not args:
usage("Error: Must specify a username to delete")
elif len(args) > 1:
usage("Error: Too many arguments")
username = args[0]
else:
if not args:
usage("Error: Must specify a username")
elif len(args) > 2:
usage("Error: Too many arguments")
elif len(args) == 1:
username = args[0]
else:
username, password = args
return auth_protocol, auth_db, auth_realm, delete, username, password
def test_package_sets(self):
""" Load a standard package set configuration """
config, handler = ZConfig.loadConfig(self.schema, PACKAGES_CONFIG_FILE)
self.assertEquals(config.PackageSets.PackageSet[0].Package[0].port,
'security/sudo')
self.assertEquals(config.PackageSets.PackageSet[1].Package[0].port,
'databases/mysql50-server')
self.assertEquals(
config.PackageSets.PackageSet[1].Package[0].BuildOptions.
Options['WITH_COLLATION'], 'UTF8')
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"-d",
"--days",
dest="days",
default="0",
help="Days of history to keep (default 0)",
)
parser.add_option(
"--prepack",
dest="prepack",
default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_option(
"--use-prepack-state",
dest="reuse_prepack",
default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
t = time.time() - float(options.days) * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
log.info("Opening %s...", name)
storage = s.open()
log.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t,
ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
log.info("Packed %s.", name)
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run", dest="dry_run", action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear", dest="clear", action="store_true",
help="Clear the contents of the destination storage before copying")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' % (
TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination):
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(os.path.dirname(appsetup.__file__), 'schema',
'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError as msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
# Insert all specified Python paths
if options.path:
sys.path[:0] = [os.path.abspath(p) for p in options.path]
# Parse product configs
zope.app.appsetup.product.setProductConfigurations(options.product_config)
# Setup the event log
options.eventlog()
# Setup other defined loggers
for logger in options.loggers:
logger()
# Insert the devmode feature, if turned on
if options.devmode:
features += ('devmode', )
logging.warning(
"Developer mode is enabled: this is a security risk "
"and should NOT be enabled on production servers. Developer mode "
"can usually be turned off by setting the `devmode` option to "
"`off` or by removing it from the instance configuration file "
"completely.")
# Execute the ZCML configuration.
appsetup.config(options.site_definition, features=features)
# Connect to and open the database, notify subscribers.
db = appsetup.multi_database(options.databases)[0][0]
notify(zope.processlifetime.DatabaseOpened(db))
return db
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(
os.path.dirname(appsetup.__file__), 'schema', 'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError as msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
# Insert all specified Python paths
if options.path:
sys.path[:0] = [os.path.abspath(p) for p in options.path]
# Parse product configs
zope.app.appsetup.product.setProductConfigurations(
options.product_config)
# Setup the event log
options.eventlog()
# Setup other defined loggers
for logger in options.loggers:
logger()
# Insert the devmode feature, if turned on
if options.devmode:
features += ('devmode',)
logging.warning("Developer mode is enabled: this is a security risk "
"and should NOT be enabled on production servers. Developer mode "
"can usually be turned off by setting the `devmode` option to "
"`off` or by removing it from the instance configuration file "
"completely.")
# Execute the ZCML configuration.
appsetup.config(options.site_definition, features=features)
# Connect to and open the database, notify subscribers.
db = appsetup.multi_database(options.databases)[0][0]
notify(zope.processlifetime.DatabaseOpened(db))
return db
def test_default_dists(self):
""" Ensure that default Dists are set properly """
subs = CONFIG_SUBS.copy()
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
release = config.Releases.Release[1]
self.assertEquals(release.dists, [
"base", "kernels", "doc", "games", "manpages", "catpages",
"proflibs", "dict", "info", "src"
])
self.assertEquals(release.kerneldists, ["GENERIC", "SMP"])
def setUp(self):
# Load ZConfig schema
self.schema = ZConfig.loadSchema(farb.CONFIG_SCHEMA)
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE,
CONFIG_SUBS)
self.config, handler = ZConfig.loadConfig(self.schema,
RELEASE_CONFIG_FILE)
self.instSection = self.config.Installations.Installation[0]
self.releaseSection = self.config.Releases.Release[0]
self.instSectionNoCommands = self.config.Installations.Installation[1]
self.instSectionNoDisks = self.config.Installations.Installation[2]
def test_release_packages(self):
""" Test that the release packages list contains good values """
config, handler = ZConfig.loadConfig(self.schema, PACKAGES_CONFIG_FILE)
farb.config.verifyPackages(config)
self.assertEquals(config.Releases.Release[1].packages[0].port, 'security/sudo')
# Test default handling
self.assertEquals(config.Releases.Release[1].packages[0].package, 'sudo')
# Test default override
self.assertEquals(config.Releases.Release[1].packages[1].package, 'overwrote')
# Verify that all package sets are loaded
self.assertEquals(config.Releases.Release[0].packages[2].package, 'mysql50-server')
def readConfig(self, filename):
# Read configuration file
schema_string = open(self.ZCONFIG_SCHEMA).read()
plugins = [configuration
for (configuration, handler) in plugin_configurations]
schema_string = schema_string % {'plugins': "\n".join(plugins)}
schema_file = StringIO(schema_string)
schema = ZConfig.loadSchemaFile(schema_file, self.ZCONFIG_SCHEMA)
config, handler = ZConfig.loadConfig(schema, filename)
return config, handler
def test_partition_softupdates(self):
""" Verify that SoftUpdates flags are tweaked appropriately """
bs = CONFIG_SUBS.copy()
bs['@SWAPSU@'] = 'True'
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, bs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
for part in config.Partitions.PartitionMap[0].Partition:
if (part.type == 'swap'):
self.assertEquals(part.softupdates, False)
elif (part.mount == '/usr'):
self.assertEquals(part.softupdates, True)
def test_partition_softupdates(self):
""" Verify that SoftUpdates flags are tweaked appropriately """
bs = CONFIG_SUBS.copy()
bs['@SWAPSU@'] = 'True'
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, bs)
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
for part in config.Partitions.PartitionMap[0].Partition:
if (part.type == 'swap'):
self.assertEquals(part.softupdates, False)
elif (part.mount == '/usr'):
self.assertEquals(part.softupdates, True)
def _get_database_from_zconfig(self):
settings = self.config.get_settings(self._zconfig_args)
path = settings['path']
frag = settings.get('frag', '')
schema = ZConfig.loadSchemaFile(StringIO(self._schema_xml_template))
config, _ = ZConfig.loadConfig(schema, path)
for database in config.databases:
if not frag or frag == database.name:
return database.open()
else:
raise ValueError("Database %r not found." % frag)
def readConfig(self, filename):
# Read configuration file
schema_string = open(self.ZCONFIG_SCHEMA).read()
plugins = [
configuration for (configuration, handler) in plugin_configurations
]
schema_string = schema_string % {'plugins': "\n".join(plugins)}
schema_file = StringIO(schema_string)
schema = ZConfig.loadSchemaFile(schema_file, self.ZCONFIG_SCHEMA)
config, handler = ZConfig.loadConfig(schema, filename)
return config, handler
def _get_database_from_zconfig(self):
settings = self.config.get_settings(self._zconfig_args)
from django_zodb.storage.base import norm_and_clean_path
path = norm_and_clean_path(settings['path'])
frag = settings.get('frag', '')
schema = ZConfig.loadSchemaFile(StringIO(self._schema_xml_template))
config, _ = ZConfig.loadConfig(schema, path)
for database in config.databases:
if not frag or frag == database.name:
return database.open()
else:
raise ValueError("Database %r not found." % frag)
def test_missingPartitionMap(self):
"""
Test handling of a missing PartitionMap
"""
# Break referential integrity
subs = CONFIG_SUBS.copy()
subs['@PMAP@'] = 'DoesNotExist'
# Rewrite and reload config
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
self.config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
# Kaboom?
self.assertRaises(ZConfig.ConfigurationError, config.verifyReferences, self.config)
def get_config(database=None):
conf = getGlobalConfiguration()
if database:
conf["zodb-db"] = conf["mysqldb"] = database
else:
conf["mysqldb"] = conf.get("mysqldb", conf.get("zodb-db"))
conf["zodb-db"] = conf.get("zodb-db", conf.get("mysqldb"))
zodb_socket = conf.get("mysqlsocket", conf.get("zodb-socket"))
if zodb_socket:
conf["socket"] = "unix_socket %s" % zodb_socket
else:
conf["socket"] = ""
newer_conf = {
"zodb-host": conf.get("host"),
"zodb-port": conf.get("port"),
"zodb-db": conf.get("mysqldb"),
"zodb-user": conf.get("mysqluser"),
"zodb-password": conf.get("mysqlpasswd"),
}
newer_conf.update(conf)
_storage_config = (
"""
<relstorage>
pack-gc true
keep-history false
<mysql>
host %(zodb-host)s
port %(zodb-port)s
db %(zodb-db)s
user %(zodb-user)s
passwd %(zodb-password)s
%(socket)s
</mysql>
</relstorage>
"""
% newer_conf
)
with tempfile.NamedTemporaryFile() as configfile:
configfile.write(_storage_config)
configfile.flush()
config, handler = ZConfig.loadConfig(schema, configfile.name)
return config
def test_release_packages(self):
""" Test that the release packages list contains good values """
config, handler = ZConfig.loadConfig(self.schema, PACKAGES_CONFIG_FILE)
farb.config.verifyPackages(config)
self.assertEquals(config.Releases.Release[1].packages[0].port,
'security/sudo')
# Test default handling
self.assertEquals(config.Releases.Release[1].packages[0].package,
'sudo')
# Test default override
self.assertEquals(config.Releases.Release[1].packages[1].package,
'overwrote')
# Verify that all package sets are loaded
self.assertEquals(config.Releases.Release[0].packages[2].package,
'mysql50-server')
def test_missingPackageSet(self):
"""
Test handling of a missing PackageSet
"""
# Break referential integrity
subs = CONFIG_SUBS.copy()
subs['@PSET@'] = 'DoesNotExist'
# Rewrite and reload config
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
self.config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
self.instSection = self.config.Installations.Installation[0]
# Kaboom?
self.assertRaises(ZConfig.ConfigurationError, config.verifyReferences, self.config)
def setUp(self):
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, CONFIG_SUBS)
farbconfig, handler = ZConfig.loadConfig(SCHEMA, RELEASE_CONFIG_FILE)
config.verifyPackages(farbconfig)
# Copy in each release needed
for release in RELEASE_NAMES:
rewrite_config(CDROM_INF_IN, CDROM_INF, {'@CD_VERSION_LINE@' : 'CD_VERSION = ' + release.upper()})
dest = os.path.join(BUILDROOT, release, 'releaseroot', builder.RELEASE_CD_PATH)
utils.copyRecursive(ISO_MOUNTPOINT, dest)
# The fake CD image we have is for 6.2, so we might need to rename a
# directory
os.rename(os.path.join(dest, '6.2-RELEASE'), os.path.join(dest, release.upper()))
self.pbr = runner.PackageBuildRunner(farbconfig)
self.pbr.run()
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-d", "--days", dest="days", default=0,
help="Days of history to keep (default 0)",
type=float,
)
parser.add_argument(
"--prepack", dest="prepack", default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_argument(
"--use-prepack-state", dest="reuse_prepack", default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
parser.add_argument("config_file")
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfig(schema, options.config_file)
t = time.time() - options.days * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
log.info("Opening %s...", name)
storage = s.open()
log.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t, ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
log.info("Packed %s.", name)
def get_storage(config_file):
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, dummy = ZConfig.loadConfig(schema, config_file)
if len(config.storages) < 1:
raise ValueError('No storages configured')
connection = config.storages[0]
if connection.config.keep_history:
raise RuntimeError('Packing does not support history keeping storages')
name = '%s (%s)' % ((connection.name or 'storage'),
connection.__class__.__name__)
log.info("Opening %s...", name)
storage = connection.open()
log.info("Successfully openend %s", storage.getName())
if 'PostgreSQLAdapter' not in storage.getName():
raise RuntimeError('Only PostgreSQL databases are supported')
return storage
def test_missingPartitionMap(self):
"""
Test handling of a missing PartitionMap
"""
# Break referential integrity
subs = CONFIG_SUBS.copy()
subs['@PMAP@'] = 'DoesNotExist'
# Rewrite and reload config
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
self.config, handler = ZConfig.loadConfig(self.schema,
RELEASE_CONFIG_FILE)
# Kaboom?
self.assertRaises(ZConfig.ConfigurationError, config.verifyReferences,
self.config)
def get_config(database=None):
conf = getGlobalConfiguration()
if database:
conf['zodb-db'] = conf['mysqldb'] = database
else:
conf['mysqldb'] = conf.get('mysqldb', conf.get('zodb-db'))
conf['zodb-db'] = conf.get('zodb-db', conf.get('mysqldb'))
zodb_socket = conf.get('mysqlsocket',
conf.get('zodb-socket'))
if zodb_socket:
conf['socket'] = 'unix_socket %s' % zodb_socket
else:
conf['socket'] = ''
newer_conf = {
'zodb-host': conf.get('host'),
'zodb-port': conf.get('port'),
'zodb-db': conf.get('mysqldb'),
'zodb-user': conf.get('mysqluser'),
'zodb-password': conf.get('mysqlpasswd')
}
newer_conf.update(conf)
_storage_config = """
<relstorage>
pack-gc true
keep-history false
<mysql>
host %(zodb-host)s
port %(zodb-port)s
db %(zodb-db)s
user %(zodb-user)s
passwd %(zodb-password)s
%(socket)s
</mysql>
</relstorage>
""" % newer_conf
with tempfile.NamedTemporaryFile() as configfile:
configfile.write(_storage_config)
configfile.flush()
config, handler = ZConfig.loadConfig(schema, configfile.name)
return config
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"-d", "--days", dest="days", default="0",
help="Days of history to keep (default 0)",
)
parser.add_option(
"--prepack", dest="prepack", default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_option(
"--use-prepack-state", dest="reuse_prepack", default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
t = time.time() - float(options.days) * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
log.info("Opening %s...", name)
storage = s.open()
log.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t, ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
log.info("Packed %s.", name)
def _setZConfig(self):
"""Modify the config, adding automatically generated settings"""
schemafile = pkg_resources.resource_filename('zope.app.server',
'schema.xml')
schema = ZConfig.loadSchema(schemafile)
root_options, handlers = ZConfig.loadConfig(schema,
self.zope_config_file)
# Devmode from the zope.app.server.main config, copied here for
# ease of access.
self.devmode = root_options.devmode
# The defined servers.
self.servers = root_options.servers
# The number of configured threads.
self.threads = root_options.threads
def test_releases(self):
""" Load a standard release configuration """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
# Releases tftproot
tftproot = os.path.join(config.Releases.installroot, 'tftproot')
self.assertEquals(config.Releases.tftproot, tftproot)
# Per-release settings
release = config.Releases.Release[0]
buildroot = os.path.join(config.Releases.buildroot, release.getSectionName())
chroot = os.path.join(buildroot, 'releaseroot')
portsdir = os.path.join(buildroot, 'usr', 'ports')
packagedir = os.path.join(portsdir, 'packages')
self.assertEquals(release.cvstag, 'RELENG_6_0')
self.assertEquals(release.packages, None)
self.assertEquals(release.buildroot, buildroot)
self.assertEquals(release.releaseroot, chroot)
def test_missingPackageSet(self):
"""
Test handling of a missing PackageSet
"""
# Break referential integrity
subs = CONFIG_SUBS.copy()
subs['@PSET@'] = 'DoesNotExist'
# Rewrite and reload config
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE, subs)
self.config, handler = ZConfig.loadConfig(self.schema,
RELEASE_CONFIG_FILE)
self.instSection = self.config.Installations.Installation[0]
# Kaboom?
self.assertRaises(ZConfig.ConfigurationError, config.verifyReferences,
self.config)
def _setZConfig(self):
"""Modify the config, adding automatically generated settings"""
schemafile = pkg_resources.resource_filename(
'zope.app.server', 'schema.xml')
schema = ZConfig.loadSchema(schemafile)
root_options, handlers = ZConfig.loadConfig(
schema, self.zope_config_file)
# Devmode from the zope.app.server.main config, copied here for
# ease of access.
self.devmode = root_options.devmode
# The defined servers.
self.servers = root_options.servers
# The number of configured threads.
self.threads = root_options.threads
def get_config(database=None):
conf = getGlobalConfiguration()
if database:
conf['zodb-db'] = conf['mysqldb'] = database
else:
conf['mysqldb'] = conf.get('mysqldb', conf.get('zodb-db'))
conf['zodb-db'] = conf.get('zodb-db', conf.get('mysqldb'))
zodb_socket = conf.get('mysqlsocket', conf.get('zodb-socket'))
if zodb_socket:
conf['socket'] = 'unix_socket %s' % zodb_socket
else:
conf['socket'] = ''
newer_conf = {
'zodb-host': conf.get('host'),
'zodb-port': conf.get('port'),
'zodb-db': conf.get('mysqldb'),
'zodb-user': conf.get('mysqluser'),
'zodb-password': conf.get('mysqlpasswd')
}
newer_conf.update(conf)
_storage_config = """
<relstorage>
pack-gc true
keep-history false
<mysql>
host %(zodb-host)s
port %(zodb-port)s
db %(zodb-db)s
user %(zodb-user)s
passwd %(zodb-password)s
%(socket)s
</mysql>
</relstorage>
""" % newer_conf
with tempfile.NamedTemporaryFile() as configfile:
configfile.write(_storage_config)
configfile.flush()
config, handler = ZConfig.loadConfig(schema, configfile.name)
return config
def __call__(self, uri):
(scheme, netloc, path, query, frag) = urlparse.urlsplit(uri)
# urlparse doesnt understand file URLs and stuffs everything into path
(scheme, netloc, path, query, frag) = urlparse.urlsplit('http:' + path)
path = os.path.normpath(path)
schema_xml = self.schema_xml_template
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, path)
for database in config.databases:
if not frag:
# use the first defined in the file
break
elif frag == database.name:
# match found
break
else:
raise KeyError("No database named %s found" % frag)
return (path, frag), (), {}, database.open
def test_releases(self):
""" Load a standard release configuration """
config, handler = ZConfig.loadConfig(self.schema, RELEASE_CONFIG_FILE)
# Releases tftproot
tftproot = os.path.join(config.Releases.installroot, 'tftproot')
self.assertEquals(config.Releases.tftproot, tftproot)
# Per-release settings
release = config.Releases.Release[0]
buildroot = os.path.join(config.Releases.buildroot,
release.getSectionName())
chroot = os.path.join(buildroot, 'releaseroot')
portsdir = os.path.join(buildroot, 'usr', 'ports')
packagedir = os.path.join(portsdir, 'packages')
self.assertEquals(release.cvstag, 'RELENG_6_0')
self.assertEquals(release.packages, None)
self.assertEquals(release.buildroot, buildroot)
self.assertEquals(release.releaseroot, chroot)
def __call__(self, uri):
(scheme, netloc, path, query, frag) = urlparse.urlsplit(uri)
# urlparse doesnt understand file URLs and stuffs everything into path
(scheme, netloc, path, query, frag) = urlparse.urlsplit('http:' + path)
path = os.path.normpath(path)
schema_xml = self.schema_xml_template
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, path)
for database in config.databases:
if not frag:
# use the first defined in the file
break
elif frag == database.name:
# match found
break
else:
raise KeyError("No database named %s found" % frag)
return (path, frag), (), {}, database.open
def runwsgi():
parser = argparse.ArgumentParser()
parser.add_argument('-w', '--webdav', action='store_true')
parser.add_argument('address', help='<ip>:<port>')
parser.add_argument('zope_conf', help='path to zope.conf')
parser.add_argument('--timerserver-interval',
help='Interval for timerserver',
type=float)
args = parser.parse_args()
startup = os.path.dirname(Zope2.Startup.__file__)
schema = ZConfig.loadSchema(os.path.join(startup, 'zopeschema.xml'))
conf, _ = ZConfig.loadConfig(schema, args.zope_conf)
make_wsgi_app({}, zope_conf=args.zope_conf)
if six.PY2:
from Signals.SignalHandler import SignalHandler
SignalHandler.registerHandler(signal.SIGTERM, sys.exit)
else:
import warnings
warnings.warn("zope4py3: SignalHandling not implemented!")
if args.timerserver_interval:
import Products.TimerService
Products.TimerService.timerserver.TimerServer.TimerServer(
module='Zope2',
interval=args.timerserver_interval,
)
ip, port = splitport(args.address)
port = int(port)
createServer(
app_wrapper(large_file_threshold=conf.large_file_threshold,
webdav_ports=[port] if args.webdav else ()),
listen=args.address,
logger=logging.getLogger("access"),
threads=conf.zserver_threads,
asyncore_use_poll=True,
# Prevent waitress from adding its own Via and Server response headers.
ident=None,
).run()
def setUp(self):
rewrite_config(RELEASE_CONFIG_FILE_IN, RELEASE_CONFIG_FILE,
CONFIG_SUBS)
farbconfig, handler = ZConfig.loadConfig(SCHEMA, RELEASE_CONFIG_FILE)
config.verifyPackages(farbconfig)
# Copy in each release needed
for release in RELEASE_NAMES:
rewrite_config(
CDROM_INF_IN, CDROM_INF,
{'@CD_VERSION_LINE@': 'CD_VERSION = ' + release.upper()})
dest = os.path.join(BUILDROOT, release, 'releaseroot',
builder.RELEASE_CD_PATH)
utils.copyRecursive(ISO_MOUNTPOINT, dest)
# The fake CD image we have is for 6.2, so we might need to rename a
# directory
os.rename(os.path.join(dest, '6.2-RELEASE'),
os.path.join(dest, release.upper()))
self.pbr = runner.PackageBuildRunner(farbconfig)
self.pbr.run()
def database_factory(global_conf, file_storage=None, db_definition=None):
if file_storage is not None and db_definition is not None:
raise TypeError("You may only provide a 'file_storage' or a "
"'db_definition' setting, not both.")
if file_storage is not None:
filename = os.path.join(global_conf['here'], file_storage)
db = zope.app.appsetup.database(filename)
elif db_definition is not None:
filename = os.path.join(global_conf['here'], db_definition)
schema_xml = os.path.join(os.path.dirname(__file__), 'schema.xml')
schema = ZConfig.loadSchema(schema_xml)
cfgroot, cfghandlers = ZConfig.loadConfig(schema, filename)
result, databases = multi_database(cfgroot.databases)
db = result[0]
zope.event.notify(zope.app.appsetup.DatabaseOpened(db))
else:
db = None
return db
def __call__(self, uri):
(scheme, netloc, path, query, frag) = urlsplit(uri)
if _BROKEN_URLSPLIT: #pragma NO COVER
# urlsplit used not to allow fragments in non-standard schemes,
# stuffed everything into 'path'
(scheme, netloc, path, query, frag
) = urlsplit('http:' + path)
path = os.path.normpath(path)
schema_xml = self.schema_xml_template
schema = ZConfig.loadSchemaFile(io.BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, path)
for database in config.databases:
if not frag:
# use the first defined in the file
break
elif frag == database.name:
# match found
break
else:
raise KeyError("No database named %s found" % frag)
return (path, frag), (), {}, database.open
def config(configfile, schemafile=None, features=()):
# Load the configuration schema
if schemafile is None:
schemafile = os.path.join(
os.path.dirname(appsetup.__file__), 'schema', 'schema.xml')
# Let's support both, an opened file and path
if isinstance(schemafile, basestring):
schema = ZConfig.loadSchema(schemafile)
else:
schema = ZConfig.loadSchemaFile(schemafile)
# Load the configuration file
# Let's support both, an opened file and path
try:
if isinstance(configfile, basestring):
options, handlers = ZConfig.loadConfig(schema, configfile)
else:
options, handlers = ZConfig.loadConfigFile(schema, configfile)
except ZConfig.ConfigurationError, msg:
sys.stderr.write("Error: %s\n" % str(msg))
sys.exit(2)
