def get_setup():
optinfo = CLIENT_COMMON_OPTIONS
optinfo['nouids'] = Option("Do not include UID numbers for users",
default=False,
cmd='--no-uids',
long_arg=True,
cook=get_bool)
optinfo['nogids'] = Option("Do not include GID numbers for groups",
default=False,
cmd='--no-gids',
long_arg=True,
cook=get_bool)
setup = OptionParser(optinfo)
setup.parse(sys.argv[1:])
if setup['args']:
print("posixuser_[baseline.py takes no arguments, only options")
print(setup.buildHelpMessage())
raise SystemExit(1)
level = 30
if setup['verbose']:
level = 20
if setup['debug']:
level = 0
Bcfg2.Logger.setup_logging('posixusers_baseline.py',
to_syslog=False,
level=level,
to_file=setup['logging'])
return setup
def __init__(self, setup):
super(RedisTransport, self).__init__(setup)
self._redis = None
self._commands = None
self.logger.error("Warning: RedisTransport is experimental")
if not HAS_REDIS:
self.logger.error("redis python module is not available")
raise TransportError
setup.update(
dict(
reporting_redis_host=Option('Redis Host',
default='127.0.0.1',
cf=('reporting', 'redis_host')),
reporting_redis_port=Option('Redis Port',
default=6379,
cf=('reporting', 'redis_port')),
reporting_redis_db=Option('Redis DB',
default=0,
cf=('reporting', 'redis_db')),
))
setup.reparse()
self._redis_host = setup.get('reporting_redis_host', '127.0.0.1')
try:
self._redis_port = int(setup.get('reporting_redis_port', 6379))
except ValueError:
self.logger.error("Redis port must be an integer")
raise TransportError
self._redis_db = setup.get('reporting_redis_db', 0)
self._redis = redis.Redis(host=self._redis_host,
port=self._redis_port,
db=self._redis_db)
def setUp(self):
self.options = [
Option("--cls", cf=("config", "cls"),
action=ImportComponentAction),
Option("--module", action=ImportModuleAction)
]
self.result = argparse.Namespace()
new_parser()
self.parser = get_parser(components=[self], namespace=self.result)
def setUp(self):
# parsing options can modify the Option objects themselves.
# that's probably bad -- and it's definitely bad if we ever
# want to do real on-the-fly config changes -- but it's easier
# to leave it as is and set the options on each test.
self.options = [
Option(cf=("foo", "*"), dest="all"),
Option(cf=("foo", "test*"), dest="test"),
Option(cf=("foo", "bogus*"), dest="unmatched"),
Option(cf=("bar", "*"), dest="no_section"),
Option(cf=("foo", "foo"))]
def setUp(self):
self.options = [
PathOption(cf=("test", "config2"), action=ConfigFileAction),
PathOption(cf=("test", "config3"), action=ConfigFileAction),
Option(cf=("test", "foo")),
Option(cf=("test", "bar")),
Option(cf=("test", "baz"))
]
self.results = argparse.Namespace()
new_parser()
self.parser = get_parser(components=[self], namespace=self.results)
def setUp(self):
self.options = [
WildcardSectionGroup(Option(cf=("four:*", "foo")),
Option(cf=("four:*", "bar"))),
WildcardSectionGroup(Option(cf=("five:*", "foo")),
Option(cf=("five:*", "bar")),
prefix="",
dest="sections")
]
self.results = argparse.Namespace()
new_parser()
self.parser = get_parser(components=[self], namespace=self.results)
def test_anchored_regex_list(self):
"""parse regex lists."""
self.options = [Option("--test", type=Types.anchored_regex_list)]
self.assertItemsEqual(
[r.pattern for r in self._test_options(["--test", r'\d+ \s*'])],
[r'^\d+$', r'^\s*$'])
self.assertRaises(SystemExit, self._test_options, ["--test", '(]'])
def setUp(self):
# parsing options can modify the Option objects themselves.
# that's probably bad -- and it's definitely bad if we ever
# want to do real on-the-fly config changes -- but it's easier
# to leave it as is and set the options on each test.
OptionTestCase.setUp(self)
self.options = [
BooleanOption("--test-true-boolean",
env="TEST_TRUE_BOOLEAN",
cf=("test", "true_boolean"),
default=True),
BooleanOption("--test-false-boolean",
env="TEST_FALSE_BOOLEAN",
cf=("test", "false_boolean"),
default=False),
BooleanOption(cf=("test", "true_config_boolean"), default=True),
BooleanOption(cf=("test", "false_config_boolean"), default=False),
Option("--test-option",
env="TEST_OPTION",
cf=("test", "option"),
default="foo"),
PathOption("--test-path-option",
env="TEST_PATH_OPTION",
cf=("test", "path"),
default="/test")
]
def test_username(self, mock_getpwnam):
"""parse username options."""
self.options = [Option("--test", type=Types.username)]
mock_getpwnam.return_value = ("test", '********', 1001, 1001,
"Test user", "/home/test", "/bin/bash")
self.assertEqual(self._test_options(["--test", "1001"]), 1001)
self.assertEqual(self._test_options(["--test", "test"]), 1001)
def reporting_storage(cls):
""" Load a Reporting storage backend """
if cls._reporting_storage is None:
cls._reporting_storage = Option(cf=('reporting', 'storage'),
dest="reporting_storage",
help='Reporting storage engine',
action=ReportingStorageAction,
default='DjangoORM')
return cls._reporting_storage
def reporting_transport(cls):
""" Load a Reporting transport backend """
if cls._reporting_transport is None:
cls._reporting_transport = Option(cf=('reporting', 'transport'),
dest="reporting_transport",
help='Reporting transport',
action=ReportingTransportAction,
default='DirectStore')
return cls._reporting_transport
def __init__(self, *items, **kwargs):
r"""
:param \*args: Child options
:type \*args: Bcfg2.Options.Option
:param prefix: The prefix to use for options generated by this
option group. By default this is generated
automatically from the config glob; see above
for details.
:type prefix: string
:param dest: The destination for the list of known sections
that match the glob.
:param dest: string
"""
OptionContainer.__init__(self, [])
self._section_glob = items[0].cf[0]
# get a default destination
self._prefix = kwargs.get("prefix",
self._dest_re.sub('_', self._section_glob))
Option.__init__(self, dest=kwargs.get('dest',
self._prefix + "sections"))
self._options = items
def plugins(cls):
""" Load a list of Bcfg2 server plugins """
if cls._plugins is None:
cls._plugins = Option(cf=('server', 'plugins'),
type=Types.comma_list,
help="Server plugin list",
action=PluginsAction,
default=[
'Bundler', 'Cfg', 'Metadata', 'Pkgmgr',
'Rules', 'SSHbase'
])
return cls._plugins
def __init__(self, *items, **kwargs):
r"""
:param \*args: Child options
:type \*args: Bcfg2.Options.Option
:param prefix: The prefix to use for options generated by this
option group. By default this is generated
automatically from the config glob; see above
for details.
:type prefix: string
:param dest: The destination for the list of known sections
that match the glob.
:param dest: string
"""
_OptionContainer.__init__(self, [])
self._section_glob = items[0].cf[0]
# get a default destination
self._prefix = kwargs.get("prefix",
self._dest_re.sub('_', self._section_glob))
Option.__init__(self,
dest=kwargs.get('dest', self._prefix + "sections"))
self.option_templates = items
def test_comma_list(self):
"""parse comma-list values."""
self.options = [Option("--test", type=Types.comma_list)]
expected = ["one", "two", "three"]
self.assertItemsEqual(self._test_options(["--test", "one,two,three"]),
expected)
self.assertItemsEqual(
self._test_options(["--test", "one, two, three"]), expected)
self.assertItemsEqual(
self._test_options(["--test", "one , two ,three"]), expected)
self.assertItemsEqual(self._test_options(["--test", "one two, three"]),
["one two", "three"])
def setUp(self):
OptionTestCase.setUp(self)
self.options = [
Option("--parent",
type=Types.comma_list,
default=["one", "two"],
action=ParentComponentAction)
]
self.result = argparse.Namespace()
new_parser()
self.parser = get_parser(components=[self],
namespace=self.result,
description="component testing parser")
def _test_dest(self, *args, **kwargs):
"""helper to test that ``dest`` is set properly."""
args = list(args)
expected = args.pop(0)
config_file = args.pop()
sentinel = object()
kwargs["default"] = sentinel
result = argparse.Namespace()
parser = Parser(namespace=result)
parser.add_options([Option(*args, **kwargs)])
parser.parse(["-C", config_file])
self.assertTrue(hasattr(result, expected))
self.assertEqual(getattr(result, expected), sentinel)
def filemonitor(cls):
""" Load a single Bcfg2 file monitor (from
:attr:`Bcfg2.Server.FileMonitor.available`) """
if cls._filemonitor is None:
import Bcfg2.Server.FileMonitor
class FileMonitorAction(ComponentAction):
""" ComponentAction for loading a single FAM backend
class """
islist = False
mapping = Bcfg2.Server.FileMonitor.available
cls._filemonitor = Option(cf=('server', 'filemonitor'),
action=FileMonitorAction,
default='default',
help='Server file monitoring driver')
return cls._filemonitor
def test_literal_dict(self):
"""parse literal-dict values."""
self.options = [Option("--test", type=Types.literal_dict)]
expected = {
"one": True,
"two": 2,
"three": "three",
"four": False,
"five": {
"a": 1,
"b": 2
}
}
self.assertDictEqual(
self._test_options([
"--test", '''{ "one": True, "two": 2,
"three": "three", "four": False,
"five": { "a": 1, "b": 2 }}'''
]), expected)
def test_duplicate_env_option(self):
"""add duplicate environment option."""
parser = Parser(components=[self])
self.assertRaises(OptionParserException, parser.add_options,
[Option(env="TEST_OPTION")])
def test_duplicate_cf_option(self):
"""add duplicate config file option."""
parser = Parser(components=[self])
self.assertRaises(OptionParserException, parser.add_options,
[Option(cf=("test", "option"))])
class Two(object):
"""Test class for component loading."""
options = [
Option('--test', cf=("config", "test"), dest="test", default="bar")
]
class Common(object):
""" Common options used in multiple different contexts. """
_plugins = None
_filemonitor = None
_reporting_storage = None
_reporting_transport = None
@classproperty
def plugins(cls):
""" Load a list of Bcfg2 server plugins """
if cls._plugins is None:
cls._plugins = Option(cf=('server', 'plugins'),
type=Types.comma_list,
help="Server plugin list",
action=PluginsAction,
default=[
'Bundler', 'Cfg', 'Metadata', 'Pkgmgr',
'Rules', 'SSHbase'
])
return cls._plugins
@classproperty
def filemonitor(cls):
""" Load a single Bcfg2 file monitor (from
:attr:`Bcfg2.Server.FileMonitor.available`) """
if cls._filemonitor is None:
import Bcfg2.Server.FileMonitor
class FileMonitorAction(ComponentAction):
""" ComponentAction for loading a single FAM backend
class """
islist = False
mapping = Bcfg2.Server.FileMonitor.available
cls._filemonitor = Option(cf=('server', 'filemonitor'),
action=FileMonitorAction,
default='default',
help='Server file monitoring driver')
return cls._filemonitor
@classproperty
def reporting_storage(cls):
""" Load a Reporting storage backend """
if cls._reporting_storage is None:
cls._reporting_storage = Option(cf=('reporting', 'storage'),
dest="reporting_storage",
help='Reporting storage engine',
action=ReportingStorageAction,
default='DjangoORM')
return cls._reporting_storage
@classproperty
def reporting_transport(cls):
""" Load a Reporting transport backend """
if cls._reporting_transport is None:
cls._reporting_transport = Option(cf=('reporting', 'transport'),
dest="reporting_transport",
help='Reporting transport',
action=ReportingTransportAction,
default='DirectStore')
return cls._reporting_transport
#: Set the path to the Bcfg2 repository
repository = _repository_option
#: Daemonize process, storing PID
daemon = PathOption('-D',
'--daemon',
help="Daemonize process, storing PID")
#: Run interactively, prompting the user for each change
interactive = BooleanOption(
"-I",
"--interactive",
help='Run interactively, prompting the user for each change')
#: Log to syslog
syslog = BooleanOption(cf=('logging', 'syslog'),
help="Log to syslog",
default=True)
#: Server location
location = Option('-S',
'--server',
cf=('components', 'bcfg2'),
default='https://localhost:6789',
metavar='<https://server:port>',
help="Server location")
#: Communication password
password = Option('-x',
'--password',
cf=('communication', 'password'),
metavar='<password>',
help="Communication Password")
#: Path to SSL key
ssl_key = PathOption('--ssl-key',
cf=('communication', 'key'),
dest="key",
help='Path to SSL key',
default="/etc/pki/tls/private/bcfg2.key")
#: Path to SSL certificate
ssl_cert = PathOption(cf=('communication', 'certificate'),
dest="cert",
help='Path to SSL certificate',
default="/etc/pki/tls/certs/bcfg2.crt")
#: Path to SSL CA certificate
ssl_ca = PathOption(cf=('communication', 'ca'), help='Path to SSL CA Cert')
#: Default Path paranoid setting
default_paranoid = Option(cf=('mdata', 'paranoid'),
dest="default_paranoid",
default='true',
choices=['true', 'false'],
help='Default Path paranoid setting')
#: Client timeout
client_timeout = Option("-t",
"--timeout",
type=float,
default=90.0,
dest="client_timeout",
cf=('communication', 'timeout'),
help='Set the client XML-RPC timeout')
class One(MockSubcommand):
"""fake subcommand for testing."""
options = [Option("--test-one")]
class Parser(argparse.ArgumentParser):
""" The Bcfg2 option parser. Most interfaces should not need to
instantiate a parser, but should instead use
:func:`Bcfg2.Options.get_parser` to get the parser that already
exists."""
#: Option for specifying the path to the Bcfg2 config file
configfile = PathOption('-C',
'--config',
env="BCFG2_CONFIG_FILE",
help="Path to configuration file",
default="/etc/bcfg2.conf")
#: Verbose version string that is printed if executed with --version
_version_string = "%s %s on Python %s" % (os.path.basename(
sys.argv[0]), __version__, ".".join(
str(v) for v in sys.version_info[0:3]))
#: Builtin options that apply to all commands
options = [
configfile,
Option('--version',
help="Print the version and exit",
action="version",
version=_version_string),
Option('-E',
'--encoding',
metavar='<encoding>',
default='UTF-8',
help="Encoding of config files",
cf=('components', 'encoding'))
]
#: Flag used in unit tests to disable actual config file reads
unit_test = False
def __init__(self, **kwargs):
""" See :class:`argparse.ArgumentParser` for a full list of
accepted parameters.
In addition to supporting all arguments and keyword arguments
from :class:`argparse.ArgumentParser`, several additional
keyword arguments are allowed.
:param components: A list of components to add to the parser.
:type components: list
:param namespace: The namespace to store options in. Default
is :attr:`Bcfg2.Options.setup`.
:type namespace: argparse.Namespace
:param add_base_options: Whether or not to add the options in
:attr:`Bcfg2.Options.Parser.options`
to the parser. Setting this to False
is default for subparsers. Default is
True.
:type add_base_options: bool
"""
self._cfp = ConfigParser.ConfigParser()
components = kwargs.pop('components', [])
#: The namespace options will be stored in.
self.namespace = kwargs.pop('namespace', setup)
if self.namespace is None:
self.namespace = setup
add_base_options = kwargs.pop('add_base_options', True)
#: Flag to indicate that this is the pre-parsing 'early' run
#: for important options like database settings that must be
#: loaded before other components can be.
self._early = kwargs.pop('early', False)
if 'add_help' not in kwargs:
kwargs['add_help'] = add_base_options
argparse.ArgumentParser.__init__(self, **kwargs)
#: Whether or not parsing has completed on all current options.
self.parsed = False
#: The argument list that was parsed.
self.argv = None
#: Components that have been added to the parser
self.components = []
#: Options that have been added to the parser
self.option_list = []
self._defaults_set = []
self._config_files = []
if add_base_options:
self.add_component(self)
if components:
for component in components:
self.add_component(component)
def _check_duplicate_cf(self, option):
"""Check for a duplicate config file option."""
def add_options(self, options):
""" Add an explicit list of options to the parser. When
possible, prefer :func:`Bcfg2.Options.Parser.add_component` to
add a whole component instead."""
_debug("Adding options: %s" % options)
self.parsed = False
for option in options:
if option not in self.option_list:
# check for duplicates
if (hasattr(option, "env") and option.env
and option.env in [o.env for o in self.option_list]):
raise OptionParserException(
"Duplicate environment variable option: %s" %
option.env)
if (hasattr(option, "cf") and option.cf
and option.cf in [o.cf for o in self.option_list]):
raise OptionParserException(
"Duplicate config file option: %s" % (option.cf, ))
self.option_list.extend(option.list_options())
option.add_to_parser(self)
for opt in option.list_options():
opt.default_from_config(self._cfp)
self._defaults_set.append(opt)
def add_component(self, component):
""" Add a component (and all of its options) to the
parser. """
if component not in self.components:
_debug("Adding component %s to %s" % (component, self))
self.components.append(component)
if hasattr(component, "options"):
self.add_options(getattr(component, "options"))
def _set_defaults_from_config(self):
""" Set defaults from the config file for all options that can
come from the config file, but haven't yet had their default
set """
_debug("Setting defaults on all options")
for opt in self.option_list:
if opt not in self._defaults_set:
opt.default_from_config(self._cfp)
self._defaults_set.append(opt)
def _parse_config_options(self):
""" populate the namespace with default values for any options
that aren't already in the namespace (i.e., options without
CLI arguments) """
_debug("Parsing config file-only options")
for opt in self.option_list[:]:
if not opt.args and opt.dest not in self.namespace:
value = opt.default
if value:
for _, action in opt.actions.items():
_debug("Setting config file-only option %s to %s" %
(opt, value))
action(self, self.namespace, value)
else:
_debug("Setting config file-only option %s to %s" %
(opt, value))
setattr(self.namespace, opt.dest, value)
def _finalize(self):
""" Finalize the value of any options that require that
additional post-processing step. (Mostly
:class:`Bcfg2.Options.Actions.ComponentAction` subclasses.)
"""
_debug("Finalizing options")
for opt in self.option_list[:]:
opt.finalize(self.namespace)
def _reset_namespace(self):
""" Delete all options from the namespace except for a few
predefined values and config file options. """
self.parsed = False
_debug("Resetting namespace")
for attr in dir(self.namespace):
if (not attr.startswith("_")
and attr not in ['uri', 'version', 'name']
and attr not in self._config_files):
_debug("Deleting %s" % attr)
delattr(self.namespace, attr)
def _parse_early_options(self):
"""Parse early options.
Early options are options that need to be parsed before other
options for some reason. These fall into two basic cases:
1. Database options, which need to be parsed so that Django
modules can be imported, since Django configuration is all
done at import-time;
2. The repository (``-Q``) option, so that ``<repository>``
macros in other options can be resolved.
"""
_debug("Option parsing phase 2: Parse early options")
early_opts = argparse.Namespace()
early_parser = Parser(add_help=False, namespace=early_opts, early=True)
# add the repo option so we can resolve <repository>
# macros
early_parser.add_options([repository])
early_components = []
for component in self.components:
if getattr(component, "parse_first", False):
early_components.append(component)
early_parser.add_component(component)
early_parser.parse(self.argv)
_debug("Fixing up <repository> macros in early options")
for attr_name in dir(early_opts):
if not attr_name.startswith("_"):
attr = getattr(early_opts, attr_name)
if hasattr(attr, "replace"):
setattr(
early_opts, attr_name,
attr.replace("<repository>", early_opts.repository))
_debug("Early parsing complete, calling hooks")
for component in early_components:
if hasattr(component, "component_parsed_hook"):
_debug("Calling component_parsed_hook on %s" % component)
getattr(component, "component_parsed_hook")(early_opts)
_debug("Calling early parsing hooks; early options: %s" % early_opts)
for option in self.option_list:
option.early_parsing_hook(early_opts)
def add_config_file(self, dest, cfile, reparse=True):
""" Add a config file, which triggers a full reparse of all
options. """
if dest not in self._config_files:
_debug("Adding new config file %s for %s" % (cfile, dest))
self._reset_namespace()
self._cfp.read([cfile])
self._defaults_set = []
self._set_defaults_from_config()
if reparse:
self._parse_config_options()
self._config_files.append(dest)
def reparse(self, argv=None):
""" Reparse options after they have already been parsed.
:param argv: The argument list to parse. By default,
:attr:`Bcfg2.Options.Parser.argv` is reused.
(I.e., the argument list that was initially
parsed.)
:type argv: list
"""
_debug("Reparsing all options")
self._reset_namespace()
self.parse(argv or self.argv)
def parse(self, argv=None):
""" Parse options.
:param argv: The argument list to parse. By default,
``sys.argv[1:]`` is used. This is stored in
:attr:`Bcfg2.Options.Parser.argv` for reuse by
:func:`Bcfg2.Options.Parser.reparse`.
:type argv: list
"""
_debug("Parsing options")
if argv is None:
argv = sys.argv[1:] # pragma: nocover
if self.parsed and self.argv == argv:
_debug("Returning already parsed namespace")
return self.namespace
self.argv = argv
# phase 1: get and read config file
_debug("Option parsing phase 1: Get and read main config file")
bootstrap_parser = argparse.ArgumentParser(add_help=False)
self.configfile.add_to_parser(bootstrap_parser)
self.configfile.default_from_config(self._cfp)
bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0]
# check whether the specified bcfg2.conf exists
if not self.unit_test and not os.path.exists(bootstrap.config):
sys.stderr.write("Could not read %s\n" % bootstrap.config)
self.add_config_file(self.configfile.dest,
bootstrap.config,
reparse=False)
# phase 2: re-parse command line for early options; currently,
# that's database options
if not self._early:
self._parse_early_options()
else:
_debug("Skipping parsing phase 2 in early mode")
# phase 3: re-parse command line, loading additional
# components, until all components have been loaded. On each
# iteration, set defaults from config file/environment
# variables
_debug("Option parsing phase 3: Main parser loop")
# _set_defaults_from_config must be called before _parse_config_options
# This is due to a tricky interaction between the two methods:
#
# (1) _set_defaults_from_config does what its name implies, it updates
# the "default" property of each Option based on the value that exists
# in the config.
#
# (2) _parse_config_options will look at each option and set it to the
# default value that is _currently_ defined. If the option does not
# exist in the namespace, it will be added. The method carefully
# avoids overwriting the value of an option that is already defined in
# the namespace.
#
# Thus, if _set_defaults_from_config has not been called yet when
# _parse_config_options is called, all config file options will get set
# to their hardcoded defaults. This process defines the options in the
# namespace and _parse_config_options will never look at them again.
#
# we have to do the parsing in two loops: first, we squeeze as
# much data out of the config file as we can to ensure that
# all config file settings are read before we use any default
# values. then we can start looking at the command line.
while not self.parsed:
self.parsed = True
self._set_defaults_from_config()
self._parse_config_options()
self.parsed = False
remaining = []
while not self.parsed:
self.parsed = True
_debug("Parsing known arguments")
try:
_, remaining = self.parse_known_args(args=self.argv,
namespace=self.namespace)
except OptionParserException:
self.error(sys.exc_info()[1])
self._set_defaults_from_config()
self._parse_config_options()
self._finalize()
if len(remaining) and not self._early:
self.error("Unknown options: %s" % " ".join(remaining))
# phase 4: call post-parsing hooks
if not self._early:
_debug("Option parsing phase 4: Call hooks")
for component in self.components:
if hasattr(component, "options_parsed_hook"):
_debug("Calling post-parsing hook on %s" % component)
getattr(component, "options_parsed_hook")()
return self.namespace
class ConfigFileComponent(object):
"""fake component for testing component loading."""
options = [
Option("--config2", action=ConfigFileAction),
Option(cf=("config", "test"), dest="config2_test", default="bar")
]
class ComponentTwo(object):
"""fake component for testing component loading."""
options = [Option("--child", default="one", action=ChildComponentAction)]
class Two(MockSubcommand):
"""fake subcommand for testing."""
options = [Option("--test-two")]
def add_to_parser(self, parser):
Option.add_to_parser(self, parser)
_OptionContainer.add_to_parser(self, parser)
def add_to_parser(self, parser):
Option.add_to_parser(self, parser)
OptionContainer.add_to_parser(self, parser)
class ChildTwo(object):
"""fake component for testing component loading."""
options = [Option("--child-two")]
class ChildOne(object):
"""fake component for testing component loading."""
options = [Option("--child-one")]