def query(data, **options):
"""
Filter data with given JMESPath expression.
See also: https://github.com/jmespath/jmespath.py and http://jmespath.org.
:param data: Target object (a dict or a dict-like object) to query
:param options:
Keyword option may include 'ac_query' which is a string represents
JMESPath expression.
:return: Maybe queried result data, primitive (int, str, ...) or dict
"""
expression = options.get("ac_query", None)
if expression is None or not expression:
return data
try:
pexp = jmespath.compile(expression)
return pexp.search(data)
except ValueError as exc: # jmespath.exceptions.*Error inherit from it.
LOGGER.warning("Failed to compile or search: exp=%s, exc=%r",
expression, exc)
except (NameError, AttributeError):
LOGGER.warning("Filter module (jmespath) is not available. "
"Do nothing.")
return data
def _maybe_validated(cnf, schema, format_checker=None, **options):
"""
:param cnf: Configuration object
:param schema: JSON schema object
:param format_checker: A format property checker object of which class is
inherited from jsonschema.FormatChecker, it's default if None given.
:param options: Keyword options such as:
- ac_namedtuple: Convert result to nested namedtuple object if True
:return: Given `cnf` as it is if validation succeeds else None
"""
valid = True
if schema:
(valid, msg) = validate(cnf, schema, format_checker=format_checker,
safe=True)
if msg:
LOGGER.warning(msg)
if valid:
if options.get("ac_namedtuple", False):
return anyconfig.mdicts.convert_to(cnf, ac_namedtuple=True)
else:
return cnf
return None
def _maybe_validated(cnf, schema, format_checker=None, **options):
"""
:param cnf: Configuration object
:param schema: JSON schema object
:param format_checker: A format property checker object of which class is
inherited from jsonschema.FormatChecker, it's default if None given.
:param options: Keyword options such as:
- ac_namedtuple: Convert result to nested namedtuple object if True
:return: Given `cnf` as it is if validation succeeds else None
"""
valid = True
if schema:
(valid, msg) = validate(cnf,
schema,
format_checker=format_checker,
safe=True)
if msg:
LOGGER.warning(msg)
if valid:
if options.get("ac_namedtuple", False):
return anyconfig.mdicts.convert_to(cnf, ac_namedtuple=True)
else:
return cnf
return None
def find_loader(path_or_stream, parser_or_type=None, is_path_=False):
"""
Find out parser object appropriate to load configuration from a file of
given path or file or file-like object.
:param path_or_stream: Configuration file path or file or file-like object
:param parser_or_type:
Forced configuration parser type or parser object itself
:param is_path_: Specify True if given `path_or_stream` is a file path
:return:
An instance of a class inherits :class:`~anyconfig.backend.base.Parser`
or None
"""
if anyconfig.backends.is_parser(parser_or_type):
return parser_or_type
try:
psr = anyconfig.backends.find_parser(path_or_stream,
forced_type=parser_or_type,
is_path_=is_path_)
LOGGER.debug("Using config parser: %r [%s]", psr, psr.type())
return psr() # TBD: Passing initialization arguments.
except (ValueError, UnknownParserTypeError, UnknownFileTypeError):
raise
def query(data, **options):
"""
Filter data with given JMESPath expression.
See also: https://github.com/jmespath/jmespath.py and http://jmespath.org.
:parae data: Target object (a dict or a dict-like object) to query
:param options:
Keyword option may include 'ac_query' which is a string represents
JMESPath expression.
:return: Maybe queried result data, primitive (int, str, ...) or dict
"""
expression = options.get("ac_query", None)
if expression is None or not expression:
return data
try:
pexp = jmespath.compile(expression)
return pexp.search(data)
except ValueError as exc: # jmespath.exceptions.*Error inherit from it.
LOGGER.warning("Failed to compile or search: exp=%s, exc=%r",
expression, exc)
except (NameError, AttributeError):
LOGGER.warning("Filter module (jmespath) is not available. "
"Do nothing.")
return data
def mk_dump_dir_if_not_exist(f):
"""
Make dir to dump f if that dir does not exist.
:param f: path of file to dump
"""
dumpdir = os.path.dirname(f)
if not os.path.exists(dumpdir):
logging.debug("Creating output dir as it's not found: " + dumpdir)
os.makedirs(dumpdir)
def mk_dump_dir_if_not_exist(f):
"""
Make dir to dump f if that dir does not exist.
:param f: path of file to dump
"""
dumpdir = os.path.dirname(f)
if not os.path.exists(dumpdir):
logging.debug("Creating output dir as it's not found: " + dumpdir)
os.makedirs(dumpdir)
def loads(content,
ac_parser=None,
ac_dict=None,
ac_template=False,
ac_context=None,
**options):
"""
:param content: Configuration file's content (a string)
:param ac_parser: Forced parser type or ID or parser object
:param ac_dict:
callable (function or class) to make mapping object will be returned as
a result or None. If not given or ac_dict is None, default mapping
object used to store resutls is dict or
:class:`collections.OrderedDict` if ac_ordered is True and selected
backend can keep the order of items in mapping objects.
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: Context dict to instantiate template
:param options:
Optional keyword arguments. See also the description of 'options' in
:func:`single_load` function.
:return: Mapping object or any query result might be primitive objects
:raises: ValueError, UnknownProcessorTypeError
"""
if ac_parser is None:
LOGGER.warning("ac_parser was not given but it's must to find correct "
"parser to load configurations from string.")
return None
psr = find(None, forced_type=ac_parser)
schema = None
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
options["ac_schema"] = None
schema = loads(ac_schema,
ac_parser=psr,
ac_dict=ac_dict,
ac_template=ac_template,
ac_context=ac_context,
**options)
if ac_template:
compiled = anyconfig.template.try_render(content=content,
ctx=ac_context,
**options)
if compiled is not None:
content = compiled
cnf = psr.loads(content, ac_dict=ac_dict, **options)
cnf = _try_validate(cnf, schema, **options)
return _try_query(cnf, options.get("ac_query", False), **options)
def find_loader(path_or_stream, parser_or_type=None, is_path_=False):
"""
Find out config parser object appropriate to load from a file of given path
or file/file-like object.
:param path_or_stream: Configuration file path or file / file-like object
:param parser_or_type: Forced configuration parser type or parser object
:param is_path_: True if given `path_or_stream` is a file path
:return: Config parser instance or None
"""
if anyconfig.backends.is_parser(parser_or_type):
return parser_or_type
(psr, err) = anyconfig.backends.find_parser(path_or_stream, parser_or_type,
is_path_=is_path_)
if err:
LOGGER.error(err)
if psr is None:
if parser_or_type is None:
LOGGER.warning("No parser (type) was given!")
else:
LOGGER.warning("Parser %s was not found!", str(parser_or_type))
return None
LOGGER.debug("Using config parser: %r [%s]", psr, psr.type())
return psr() # TBD: Passing initialization arguments.
def find_loader(path_or_stream, parser_or_type=None, is_path_=False):
"""
Find out config parser object appropriate to load from a file of given path
or file/file-like object.
:param path_or_stream: Configuration file path or file / file-like object
:param parser_or_type: Forced configuration parser type or parser object
:param is_path_: True if given `path_or_stream` is a file path
:return: Config parser instance or None
"""
if anyconfig.backends.is_parser(parser_or_type):
return parser_or_type
(psr, err) = anyconfig.backends.find_parser(path_or_stream,
parser_or_type,
is_path_=is_path_)
if err:
LOGGER.error(err)
if psr is None:
if parser_or_type is None:
LOGGER.warning("No parser (type) was given!")
else:
LOGGER.warning("Parser %s was not found!", str(parser_or_type))
return None
LOGGER.debug("Using config parser: %r [%s]", psr, psr.type())
return psr() # TBD: Passing initialization arguments.
def dump(data, path_or_stream, ac_parser=None, **options):
"""
Save `data` as `path_or_stream`.
:param data: A mapping object may have configurations data to dump
:param path_or_stream: Output file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
"""
dumper = _find_dumper(path_or_stream, ac_parser)
LOGGER.info("Dumping: %s",
anyconfig.utils.get_path_from_stream(path_or_stream))
dumper.dump(data, path_or_stream, **options)
def dump(data, config_path, forced_type=None, **kwargs):
"""
Save `data` as `config_path`.
:param data: Config data object to dump ::
anyconfig.mergeabledict.MergeableDict by default
:param config_path: Output filename
:param forced_type: Forced configuration parser type
:param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
JSON loader/dumper backend
"""
dumper = _find_dumper(config_path, forced_type)
LOGGER.info("Dumping: %s", config_path)
dumper.dump(data, config_path, **kwargs)
def _find_dumper(path_or_stream, ac_parser=None):
"""
Find configuration parser to dump data.
:param path_or_stream: Output file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:return: Parser-inherited class object
"""
psr = find_loader(path_or_stream, ac_parser)
if psr is None or not getattr(psr, "dump", False):
LOGGER.warning("Dump method not implemented. Fallback to json.Parser")
psr = anyconfig.backend.json.Parser()
return psr
def _validate(config, schema, format_checker=None):
"""
Wrapper function for anyconfig.schema.vaildate.
:param config: Configuration object :: container
:param schema: JSON schema object :: container
:param format_checker: A format property checker object of which class is
inherited from jsonschema.FormatChecker, it's default if None given.
:return: True if validation suceeds or jsonschema module is not available.
"""
(ret, msg) = validate(config, schema, format_checker, True)
if msg:
LOGGER.warn(msg)
return ret
def _find_dumper(config_path, forced_type=None):
"""
Find configuration parser to dump data.
:param config_path: Output filename
:param forced_type: Forced configuration parser type
:return: Parser-inherited class object
"""
cparser = find_loader(config_path, forced_type)
if cparser is None or not getattr(cparser, "dump", False):
LOGGER.warn("Dump method not implemented. Fallback to json.Parser")
cparser = anyconfig.backend.json.Parser()
return cparser
def _find_dumper(path_or_stream, ac_parser=None):
"""
Find configuration parser to dump data.
:param path_or_stream: Output file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:return: Parser-inherited class object
"""
psr = find_loader(path_or_stream, ac_parser)
if psr is None or not getattr(psr, "dump", False):
LOGGER.warning("Dump method not implemented. Fallback to json.Parser")
psr = anyconfig.backend.json.Parser()
return psr
def dump(data, path_or_stream, ac_parser=None, **options):
"""
Save `data` as `path_or_stream`.
:param data: Config data object (dict[-like] or namedtuple) to dump
:param path_or_stream: Output file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
"""
dumper = _find_dumper(path_or_stream, ac_parser)
LOGGER.info("Dumping: %s",
anyconfig.utils.get_path_from_stream(path_or_stream))
if anyconfig.mdicts.is_namedtuple(data):
data = to_container(data, **options) # namedtuple -> dict-like
dumper.dump(data, path_or_stream, **options)
def dump(data, path_or_stream, ac_parser=None, **options):
"""
Save `data` as `path_or_stream`.
:param data: Config data object (dict[-like] or namedtuple) to dump
:param path_or_stream: Output file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
"""
dumper = _find_dumper(path_or_stream, ac_parser)
LOGGER.info("Dumping: %s",
anyconfig.utils.get_path_from_stream(path_or_stream))
if anyconfig.mdicts.is_namedtuple(data):
data = to_container(data, **options) # namedtuple -> dict-like
dumper.dump(data, path_or_stream, **options)
def loads(config_content, forced_type=None, ac_template=False, ac_context=None, ac_schema=None, **kwargs):
"""
:param config_content: Configuration file's content
:param forced_type: Forced configuration parser type
:param ignore_missing: Ignore missing config files
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: Context dict to instantiate template
:param ac_schema: JSON schema content to validate given config file
:param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
JSON loader/dumper backend
:return: Dict-like object (instance of
anyconfig.mergeabledict.MergeableDict by default) supports merge
operations.
"""
if forced_type is None:
LOGGER.warn("No config type was given. Try to parse...")
return anyconfig.parser.parse(config_content)
cparser = find_loader(None, forced_type)
if cparser is None:
return anyconfig.parser.parse(config_content)
if ac_schema is not None:
kwargs["ac_schema"] = None # Avoid infinit loop
format_checker = kwargs.get("format_checker", None)
schema = loads(ac_schema, forced_type, ac_template, ac_context, **kwargs)
if ac_template:
try:
LOGGER.debug("Compiling")
config_content = anyconfig.template.render_s(config_content, ac_context)
except Exception as exc:
LOGGER.debug("Exc=%s", str(exc))
LOGGER.warn("Failed to compile and fallback to no template " "mode: '%s'", config_content[:50] + "...")
config = cparser.loads(config_content, **kwargs)
if ac_schema is not None:
if not _validate(config, schema, format_checker):
LOGGER.warn("Validation failed: schema=%s", schema)
return None
return config
def _try_validate(cnf, schema, **options):
"""
:param cnf: Mapping object represents configuration data
:param schema: JSON schema object
:param options: Keyword options passed to :func:`jsonschema.validate`
:return: Given 'cnf' as it is if validation succeeds else None
"""
valid = True
if schema:
(valid, msg) = validate(cnf, schema, **options)
if msg:
LOGGER.warning(msg)
if valid:
return cnf
return None
def _try_validate(cnf, schema, **options):
"""
:param cnf: Mapping object represents configuration data
:param schema: JSON schema object
:param options: Keyword options passed to :func:`~jsonschema.validate`
:return: Given `cnf` as it is if validation succeeds else None
"""
valid = True
if schema:
(valid, msg) = validate(cnf, schema, **options)
if msg:
LOGGER.warning(msg)
if valid:
return cnf
return None
def find_loader(path_or_stream, forced_type=None, is_path_=None):
"""
Find out config parser object appropriate to load from a file of given path
or file/file-like object.
:param path_or_stream: Configuration file path or file / file-like object
:param forced_type: Forced configuration parser type
:param is_path_: True if given `path_or_stream` is a file path
:return: Config parser instance or None
"""
(psr, err) = anyconfig.backends.find_parser(path_or_stream, forced_type, is_path_=is_path_)
if psr is None:
LOGGER.error(err)
return None
LOGGER.debug("Using config parser of type: %s", psr.type())
return psr() # TBD: Passing initialization arguments.
def loads(content, ac_parser=None, ac_dict=None, ac_template=False,
ac_context=None, **options):
"""
:param content: Configuration file's content (a string)
:param ac_parser: Forced parser type or ID or parser object
:param ac_dict:
callable (function or class) to make mapping object will be returned as
a result or None. If not given or ac_dict is None, default mapping
object used to store resutls is dict or
:class:`collections.OrderedDict` if ac_ordered is True and selected
backend can keep the order of items in mapping objects.
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: Context dict to instantiate template
:param options:
Optional keyword arguments. See also the description of 'options' in
:func:`single_load` function.
:return: Mapping object or any query result might be primitive objects
:raises: ValueError, UnknownProcessorTypeError
"""
if ac_parser is None:
LOGGER.warning("ac_parser was not given but it's must to find correct "
"parser to load configurations from string.")
return None
psr = find(None, forced_type=ac_parser)
schema = None
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
options["ac_schema"] = None
schema = loads(ac_schema, ac_parser=psr, ac_dict=ac_dict,
ac_template=ac_template, ac_context=ac_context,
**options)
if ac_template:
compiled = anyconfig.template.try_render(content=content,
ctx=ac_context, **options)
if compiled is not None:
content = compiled
cnf = psr.loads(content, ac_dict=ac_dict, **options)
cnf = _try_validate(cnf, schema, **options)
return anyconfig.query.query(cnf, **options)
def _single_load(input_,
ac_parser=None,
ac_template=False,
ac_context=None,
**options):
"""
:param input_:
File path or file or file-like object or pathlib.Path object represents
the file or a namedtuple 'anyconfig.globals.IOInfo' object represents
some input to load some data from
:param ac_parser: Forced parser type or parser object itself
:param ac_template:
Assume configuration file may be a template file and try to compile it
AAR if True
:param ac_context: A dict presents context to instantiate template
:param options:
Optional keyword arguments :func:`single_load` supports except for
ac_schema and ac_query
:return: Mapping object
:raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError
"""
ioi = anyconfig.ioinfo.make(input_)
psr = find(ioi, forced_type=ac_parser)
filepath = ioi.path
# .. note::
# This will be kept for backward compatibility until 'ignore_missing'
# option is deprecated and removed completely.
if "ignore_missing" in options:
warnings.warn(
"keyword option 'ignore_missing' is deprecated, use "
"'ac_ignore_missing' instead", DeprecationWarning)
options["ac_ignore_missing"] = options["ignore_missing"]
LOGGER.info("Loading: %s", filepath)
if ac_template and filepath is not None:
content = anyconfig.template.try_render(filepath=filepath,
ctx=ac_context,
**options)
if content is not None:
return psr.loads(content, **options)
return psr.load(ioi, **options)
def _maybe_schema(**options):
"""
:param options: Optional keyword arguments such as
- ac_template: Assume configuration file may be a template file and try
to compile it AAR if True
- ac_context: Mapping object presents context to instantiate template
- ac_schema: JSON schema file path to validate configuration files
"""
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
# Try to detect the appropriate as it may be different from the
# original config file's format, perhaps.
options["ac_parser"] = None
options["ac_schema"] = None # Avoid infinite loop.
LOGGER.info("Loading schema: %s", ac_schema)
return load(ac_schema, **options)
return None
def dump(data, path_or_stream, forced_type=None, **kwargs):
"""
Save `data` as `path_or_stream`.
:param data: Config data object to dump ::
anyconfig.mergeabledict.MergeableDict by default
:param path_or_stream: Output file path or file / file-like object
:param forced_type: Forced configuration parser type
:param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
JSON loader/dumper backend
"""
dumper = _find_dumper(path_or_stream, forced_type)
if _is_path(path_or_stream):
LOGGER.info("Dumping: %s", path_or_stream)
else:
LOGGER.info("Dumping: %s", get_path_from_stream(path_or_stream))
dumper.dump(data, path_or_stream, **kwargs)
def _load_schema(**options):
"""
:param options: Optional keyword arguments such as
- ac_template: Assume configuration file may be a template file and try
to compile it AAR if True
- ac_context: A dict presents context to instantiate template
- ac_schema: JSON schema file path to validate given config file
"""
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
# Try to detect the appropriate as it may be different from the
# original config file's format, perhaps.
options["ac_parser"] = None
options["ac_schema"] = None # Avoid infinite loop.
LOGGER.info("Loading schema: %s", ac_schema)
return load(ac_schema, **options)
return None
def dump(data, out, ac_parser=None, **options):
"""
Save `data` to `out`.
:param data: A mapping object may have configurations data to dump
:param out:
An output file path or a file or a file-like object or a `pathlib.Path`
object represents the file or a namedtuple `~anyconfig.globals.IOInfo`
object represents output to dump some data to.
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
:raises: ValueError, UnknownParserTypeError, UnknownFileTypeError
"""
ioi = anyconfig.backends.inspect_io_obj(out, forced_type=ac_parser)
LOGGER.info("Dumping: %s", ioi.path)
ioi.processor.dump(data, ioi, **options)
def dump(data, out, ac_parser=None, **options):
"""
Save 'data' to 'out'.
:param data: A mapping object may have configurations data to dump
:param out:
An output file path, a file, a file-like object, :class:`pathlib.Path`
object represents the file or a namedtuple 'anyconfig.globals.IOInfo'
object represents output to dump some data to.
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
:raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError
"""
ioi = anyconfig.ioinfo.make(out)
psr = find(ioi, forced_type=ac_parser)
LOGGER.info("Dumping: %s", ioi.path)
psr.dump(data, ioi, **options)
def dump(data, out, ac_parser=None, **options):
"""
Save 'data' to 'out'.
:param data: A mapping object may have configurations data to dump
:param out:
An output file path, a file, a file-like object, :class:`pathlib.Path`
object represents the file or a namedtuple 'anyconfig.globals.IOInfo'
object represents output to dump some data to.
:param ac_parser: Forced parser type or parser object
:param options:
Backend specific optional arguments, e.g. {"indent": 2} for JSON
loader/dumper backend
:raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError
"""
ioi = anyconfig.ioinfo.make(out)
psr = find(ioi, forced_type=ac_parser)
LOGGER.info("Dumping: %s", ioi.path)
psr.dump(data, ioi, **options)
def _load_impl(config_fp, sep=_SEP, **kwargs):
"""
:param config_fp: File or file-like object provides ini-style conf
:return: Dict or dict-like object represents config values
"""
config = dict()
# Optional arguements for configparser.SafeConfigParser{,readfp}
kwargs_0 = Base.mk_opt_args(("defaults", "dict_type", "allow_no_value"),
kwargs)
kwargs_1 = Base.mk_opt_args(("filename", ), kwargs)
try:
try:
parser = configparser.SafeConfigParser(**kwargs_0)
except TypeError:
# It seems ConfigPaser.*ConfigParser in python 2.6 does not support
# 'allow_no_value' option parameter, and TypeError will be thrown.
kwargs_0 = Base.mk_opt_args(("defaults", "dict_type"), kwargs)
parser = configparser.SafeConfigParser(**kwargs_0)
parser.readfp(config_fp, **kwargs_1)
if parser.defaults():
config["DEFAULT"] = dict()
for k, v in iteritems(parser.defaults()):
config["DEFAULT"][k] = _parse(v, sep)
for s in parser.sections():
config[s] = dict()
for k, v in parser.items(s):
config[s][k] = _parse(v, sep)
except Exception:
logging.warn(sys.exc_info()[-1])
raise
return config
def single_load(path_or_stream,
ac_parser=None,
ac_template=False,
ac_context=None,
**options):
"""
Load single config file.
:param path_or_stream: Configuration file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: A dict presents context to instantiate template
:param options: Optional keyword arguments such as:
- Common options:
- ac_namedtuple: Convert result to nested namedtuple object if True
- ac_schema: JSON schema file path to validate given config file
- Common backend options:
- ignore_missing: Ignore and just return empty result if given file
(``path_or_stream``) does not exist.
- Backend specific options such as {"indent": 2} for JSON backend
:return: dict or dict-like object supports merge operations
"""
is_path_ = is_path(path_or_stream)
if is_path_:
path_or_stream = anyconfig.utils.ensure_expandusr(path_or_stream)
filepath = path_or_stream
else:
filepath = anyconfig.utils.get_path_from_stream(path_or_stream)
psr = find_loader(path_or_stream, ac_parser, is_path_)
if psr is None:
return None
schema = _load_schema(ac_template=ac_template,
ac_context=ac_context,
**options)
options["ac_schema"] = None # It's not needed now.
LOGGER.info("Loading: %s", filepath)
if ac_template and filepath is not None:
try:
LOGGER.debug("Compiling: %s", filepath)
content = anyconfig.template.render(filepath, ac_context)
cnf = psr.loads(content, **options)
return _maybe_validated(cnf, schema, **options)
except Exception as exc:
LOGGER.warning(
"Failed to compile %s, fallback to no template "
"mode, exc=%r", path_or_stream, exc)
cnf = psr.load(path_or_stream, **options)
return _maybe_validated(cnf, schema, **options)
def _load_impl(config_fp, sep=_SEP, **kwargs):
"""
:param config_fp: File or file-like object provides ini-style conf
:return: Dict or dict-like object represents config values
"""
config = dict()
# Optional arguements for configparser.SafeConfigParser{,readfp}
kwargs_0 = Base.mk_opt_args(("defaults", "dict_type", "allow_no_value"),
kwargs)
kwargs_1 = Base.mk_opt_args(("filename", ), kwargs)
try:
try:
parser = configparser.SafeConfigParser(**kwargs_0)
except TypeError:
# It seems ConfigPaser.*ConfigParser in python 2.6 does not support
# 'allow_no_value' option parameter, and TypeError will be thrown.
kwargs_0 = Base.mk_opt_args(("defaults", "dict_type"), kwargs)
parser = configparser.SafeConfigParser(**kwargs_0)
parser.readfp(config_fp, **kwargs_1)
if parser.defaults():
config["DEFAULT"] = dict()
for k, v in iteritems(parser.defaults()):
config["DEFAULT"][k] = _parse(v, sep)
for s in parser.sections():
config[s] = dict()
for k, v in parser.items(s):
config[s][k] = _parse(v, sep)
except Exception:
logging.warn(sys.exc_info()[-1])
raise
return config
def _single_load(input_, ac_parser=None, ac_template=False,
ac_context=None, **options):
"""
:param input_:
File path or file or file-like object or pathlib.Path object represents
the file or a namedtuple 'anyconfig.globals.IOInfo' object represents
some input to load some data from
:param ac_parser: Forced parser type or parser object itself
:param ac_template:
Assume configuration file may be a template file and try to compile it
AAR if True
:param ac_context: A dict presents context to instantiate template
:param options:
Optional keyword arguments :func:`single_load` supports except for
ac_schema and ac_query
:return: Mapping object
:raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError
"""
ioi = anyconfig.ioinfo.make(input_)
psr = find(ioi, forced_type=ac_parser)
filepath = ioi.path
# .. note::
# This will be kept for backward compatibility until 'ignore_missing'
# option is deprecated and removed completely.
if "ignore_missing" in options:
warnings.warn("keyword option 'ignore_missing' is deprecated, use "
"'ac_ignore_missing' instead", DeprecationWarning)
options["ac_ignore_missing"] = options["ignore_missing"]
LOGGER.info("Loading: %s", filepath)
if ac_template and filepath is not None:
content = anyconfig.template.try_render(filepath=filepath,
ctx=ac_context, **options)
if content is not None:
return psr.loads(content, **options)
return psr.load(ioi, **options)
def loads(content,
ac_parser=None,
ac_template=False,
ac_context=None,
**options):
"""
:param content: Configuration file's content
:param ac_parser: Forced parser type or parser object
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: Context dict to instantiate template
:param options:
Optional keyword arguments. See also the description of `options` in
`single_load` function.
:return: dict or dict-like object supports merge operations
"""
msg = "Try parsing with a built-in parser because %s"
if ac_parser is None:
LOGGER.warning(msg, "ac_parser was not given.")
return None
psr = find_loader(None, ac_parser)
if psr is None:
LOGGER.warning(msg, "parser '%s' was not found" % ac_parser)
return None
schema = None
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
options["ac_schema"] = None
schema = loads(ac_schema,
ac_parser=psr,
ac_template=ac_template,
ac_context=ac_context,
**options)
if ac_template:
try:
LOGGER.debug("Compiling")
content = anyconfig.template.render_s(content, ac_context)
except Exception as exc:
LOGGER.warning(
"Failed to compile and fallback to no template "
"mode: '%s', exc=%r", content[:50] + '...', exc)
cnf = psr.loads(content, **options)
return _maybe_validated(cnf, schema, **options)
def single_load(path_or_stream, ac_parser=None, ac_template=False,
ac_context=None, **options):
"""
Load single config file.
:param path_or_stream: Configuration file path or file / file-like object
:param ac_parser: Forced parser type or parser object
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: A dict presents context to instantiate template
:param options: Optional keyword arguments such as:
- Common options:
- ac_namedtuple: Convert result to nested namedtuple object if True
- ac_schema: JSON schema file path to validate given config file
- Common backend options:
- ignore_missing: Ignore and just return empty result if given file
(``path_or_stream``) does not exist.
- Backend specific options such as {"indent": 2} for JSON backend
:return: dict or dict-like object supports merge operations
"""
is_path_ = is_path(path_or_stream)
if is_path_:
path_or_stream = anyconfig.utils.ensure_expandusr(path_or_stream)
filepath = path_or_stream
else:
filepath = anyconfig.utils.get_path_from_stream(path_or_stream)
psr = find_loader(path_or_stream, ac_parser, is_path_)
if psr is None:
return None
schema = _load_schema(ac_template=ac_template, ac_context=ac_context,
**options)
options["ac_schema"] = None # It's not needed now.
LOGGER.info("Loading: %s", filepath)
if ac_template and filepath is not None:
try:
LOGGER.debug("Compiling: %s", filepath)
content = anyconfig.template.render(filepath, ac_context)
cnf = psr.loads(content, **options)
return _maybe_validated(cnf, schema, **options)
except Exception as exc:
LOGGER.warning("Failed to compile %s, fallback to no template "
"mode, exc=%r", path_or_stream, exc)
cnf = psr.load(path_or_stream, **options)
return _maybe_validated(cnf, schema, **options)
def loads(content, ac_parser=None, ac_template=False, ac_context=None,
**options):
"""
:param content: Configuration file's content
:param ac_parser: Forced parser type or parser object
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: Context dict to instantiate template
:param options:
Optional keyword arguments. See also the description of `options` in
`single_load` function.
:return: dict or dict-like object supports merge operations
"""
msg = "Try parsing with a built-in parser because %s"
if ac_parser is None:
LOGGER.warning(msg, "ac_parser was not given.")
return None
psr = find_loader(None, ac_parser)
if psr is None:
LOGGER.warning(msg, "parser '%s' was not found" % ac_parser)
return None
schema = None
ac_schema = options.get("ac_schema", None)
if ac_schema is not None:
options["ac_schema"] = None
schema = loads(ac_schema, ac_parser=psr, ac_template=ac_template,
ac_context=ac_context, **options)
if ac_template:
try:
LOGGER.debug("Compiling")
content = anyconfig.template.render_s(content, ac_context)
except Exception as exc:
LOGGER.warning("Failed to compile and fallback to no template "
"mode: '%s', exc=%r", content[:50] + '...', exc)
cnf = psr.loads(content, **options)
return _maybe_validated(cnf, schema, **options)
def find_loader(config_path, forced_type=None):
"""
:param config_path: Configuration file path
:param forced_type: Forced configuration parser type
:return: Parser-inherited class object
"""
if forced_type is not None:
cparser = anyconfig.backends.find_by_type(forced_type)
if not cparser:
LOGGER.error("No parser found for given type: %s", forced_type)
return None
else:
cparser = anyconfig.backends.find_by_file(config_path)
if not cparser:
LOGGER.error("No parser found for given file: %s", config_path)
return None
LOGGER.debug("Using config parser of type: %s", cparser.type())
return cparser
import anyconfig.compat as AC
SUPPORTED = True
try:
# First, try lxml which is compatible with elementtree and looks faster a
# lot. See also: http://getpython3.com/diveintopython3/xml.html
from lxml2 import etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except ImportError:
try:
import elementtree.ElementTree as etree
except ImportError:
logging.warn("ElementTree module is not available. Disabled "
"XML support.")
SUPPORTED = False
if SUPPORTED:
def etree_getroot_fromstring(s):
"""
:param s: A XML string
:return: etree object gotten by parsing ``s``
"""
return etree.ElementTree(etree.fromstring(s)).getroot()
def etree_getroot_fromsrc(src):
"""
:param src: A file name/path or a file[-like] object or a URL
:return: etree object gotten by parsing ``s``
import anyconfig.backend.base as Base
import anyconfig.compat as AC
SUPPORTED = True
try:
# First, try lxml which is compatible with elementtree and looks faster a
# lot. See also: http://getpython3.com/diveintopython3/xml.html
from lxml2 import etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except ImportError:
try:
import elementtree.ElementTree as etree
except ImportError:
logging.warn("ElementTree module is not available. Disabled "
"XML support.")
SUPPORTED = False
if SUPPORTED:
def etree_getroot_fromstring(s):
"""
:param s: A XML string
:return: etree object gotten by parsing ``s``
"""
return etree.ElementTree(etree.fromstring(s)).getroot()
def etree_getroot_fromsrc(src):
"""
:param src: A file name/path or a file[-like] object or a URL
:return: etree object gotten by parsing ``s``
def set_loglevel(level):
"""
:param level: Log level, e.g. logging.INFO and logging.WARN.
"""
LOGGER.setLevel(level)
#
# Copyright (C) 2011 - 2013 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
from anyconfig.globals import LOGGER as logging
import anyconfig.backend.base as Base
SUPPORTED = False
try:
import yaml
SUPPORTED = True
except ImportError:
logging.warn("YAML module is not available. Disabled its support.")
if SUPPORTED:
def filter_keys(keys, filter_key):
return [k for k in keys if k != filter_key]
def yaml_load(fp, **kwargs):
keys = filter_keys(kwargs.keys(), "safe")
if kwargs.get("safe", False):
return yaml.safe_load(fp, **Base.mk_opt_args(keys, kwargs))
else:
return yaml.load(fp, **kwargs)
def yaml_dump(data, fp, **kwargs):
keys = filter_keys(kwargs.keys(), "safe")
if kwargs.get("safe", False):
return yaml.safe_dump(data, fp, **Base.mk_opt_args(keys, kwargs))
def _dummy_fun(*args, **kwargs):
logging.warn(
"Return {} as YAML module is not available: "
"args=%s, kwargs=%s", ','.join(args), str(kwargs))
return {}
def _dummy_fun(*args, **kwargs):
logging.warn("Return None as XML module is not available: "
"args=%s, kwargs=%s", ','.join(args), str(kwargs))
return None
def single_load(
config_path, forced_type=None, ignore_missing=False, ac_template=False, ac_context=None, ac_schema=None, **kwargs
):
"""
Load single config file.
:param config_path: Configuration file path
:param forced_type: Forced configuration parser type
:param ignore_missing: Ignore and just return empty result if given file
(``config_path``) does not exist
:param ac_template: Assume configuration file may be a template file and
try to compile it AAR if True
:param ac_context: A dict presents context to instantiate template
:param ac_schema: JSON schema file path to validate given config file
:param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
JSON loader/dumper backend
:return: Dict-like object (instance of
anyconfig.mergeabledict.MergeableDict by default) supports merge
operations.
"""
config_path = anyconfig.utils.ensure_expandusr(config_path)
cparser = find_loader(config_path, forced_type)
if cparser is None:
return None
if ac_schema is not None:
kwargs["ac_schema"] = None # Avoid infinit loop
format_checker = kwargs.get("format_checker", None)
LOGGER.info("Loading schema: %s", ac_schema)
schema = load(
ac_schema,
forced_type=forced_type,
ignore_missing=ignore_missing,
ac_template=ac_template,
ac_context=ac_context,
**kwargs
)
LOGGER.info("Loading: %s", config_path)
if ac_template:
try:
LOGGER.debug("Compiling: %s", config_path)
config_content = anyconfig.template.render(config_path, ac_context)
config = cparser.loads(config_content, ignore_missing=ignore_missing, **kwargs)
if ac_schema is not None:
if not _validate(config, schema, format_checker):
return None
return config
except Exception as exc:
LOGGER.debug("Exc=%s", str(exc))
LOGGER.warn("Failed to compile %s, fallback to no template " "mode", config_path)
config = cparser.load(config_path, ignore_missing=ignore_missing, **kwargs)
if ac_schema is not None:
if not _validate(config, schema, format_checker):
return None
return config
def set_loglevel(level):
"""
:param level: Log level, e.g. logging.INFO and logging.WARN.
"""
LOGGER.setLevel(level)
def single_load(path_or_stream,
ac_parser=None,
ac_template=False,
ac_context=None,
**options):
"""
Load single configuration file.
.. note::
:func:`load` is a preferable alternative and this API should be used
only if there is a need to emphasize given file path is single one.
:param path_or_stream: Configuration file path or file or file-like object
:param ac_parser: Forced parser type or parser object itself
:param ac_template:
Assume configuration file may be a template file and try to compile it
AAR if True
:param ac_context: A dict presents context to instantiate template
:param options: Optional keyword arguments such as:
- Options common with :func:`multi_load` and :func:`load`:
- ac_dict: callable (function or class) to make mapping object will
be returned as a result or None. If not given or ac_dict is None,
default mapping object used to store resutls is dict or
:class:`~collections.OrderedDict` if ac_ordered is True and
selected backend can keep the order of items in mapping objects.
- ac_ordered: True if you want to keep resuls ordered. Please note
that order of items may be lost depends on backend used.
- ac_schema: JSON schema file path to validate given config file
- ac_query: JMESPath expression to query data
- Common backend options:
- ignore_missing: Ignore and just return empty result if given file
(``path_or_stream``) does not exist.
- Backend specific options such as {"indent": 2} for JSON backend
:return: Mapping object
"""
is_path_ = is_path(path_or_stream)
if is_path_:
filepath = path_or_stream = anyconfig.utils.normpath(path_or_stream)
else:
filepath = anyconfig.utils.get_path_from_stream(path_or_stream)
psr = find_loader(path_or_stream, ac_parser, is_path_)
schema = _maybe_schema(ac_template=ac_template,
ac_context=ac_context,
**options)
LOGGER.info("Loading: %s", filepath)
if ac_template and filepath is not None:
content = anyconfig.template.try_render(filepath=filepath,
ctx=ac_context)
if content is not None:
cnf = psr.loads(content, **options)
return _maybe_validated(cnf, schema, **options)
cnf = psr.load(path_or_stream, **options)
return _maybe_validated(cnf, schema, **options)