def __load(self, rules_data: str) -> None:
"""
loads a yaml str, serializes and normalized the rule data. Then assigns the rule data to properties.
:param config_file: A :string: loaded from a yaml file.
"""
if not isinstance(rules_data, str):
raise TypeError('rules_data must be a str.')
try:
data = SafeLoader(rules_data).get_data()
if data is None:
raise AttributeError('The rules must have data in it.')
data = normalize_keys(data, snake_case=False)
for key, value in data.items():
variable_name = '_{0}'.format(key)
if hasattr(self, variable_name):
setattr(self, variable_name, value)
else:
raise AttributeError('{0} isn\'t a valid rule attribute.'.format(key))
except YAMLError as e:
if hasattr(e, 'problem_mark'):
raise SyntaxError(
"There is a syntax error in the rules line: {0} column: {1}".format(
e.problem_mark.line,
e.problem_mark.column
)
)
else:
raise SyntaxError("There is a syntax error in the rules.")
def __load(self, rules_data: str) -> None:
"""
loads a yaml str, serializes and normalized the rule data. Then assigns the rule data to properties.
:param config_file: A :string: loaded from a yaml file.
"""
if not isinstance(rules_data, str):
raise TypeError('rules_data must be a str.')
try:
data = SafeLoader(rules_data).get_data()
if data is None:
raise AttributeError('The rules must have data in it.')
data = normalize_keys(data, snake_case=False)
for key, value in data.items():
variable_name = '_{0}'.format(key)
if hasattr(self, variable_name):
setattr(self, variable_name, value)
else:
raise AttributeError(
'{0} isn\'t a valid rule attribute.'.format(key))
except YAMLError as e:
if hasattr(e, 'problem_mark'):
raise SyntaxError(
"There is a syntax error in the rules line: {0} column: {1}"
.format(e.problem_mark.line, e.problem_mark.column))
else:
raise SyntaxError("There is a syntax error in the rules.")
def __init__(self, stream):
SafeLoader.__init__(self, stream)
self.library = Library(self)
self.represent = representer.Representer().represent_data
if Loader.anchors is not None:
self.set_anchors(Loader.anchors)
if Loader.filepath is None:
self.importpath = os.getcwd()
else:
self.importpath = os.path.abspath(os.path.dirname(Loader.filepath))
def fix_yaml_loader():
"""Ensure that any string read by yaml is represented as unicode."""
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def fix_yaml_loader():
"""Ensure that any string read by yaml is represented as unicode."""
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def _load_user_settings(self):
"""Load user settings from config file."""
successful = _ensure_file(self.config_file)
if not successful:
LOG.error("Unable to load user config file.")
return
self.subscriptions = []
# Python 2/PyPy shim, per
# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
# Override the default string handling function to always return unicode objects.
def construct_yaml_str(self, node):
"""Override to force PyYAML to handle unicode on Python 2."""
return self.construct_scalar(node)
SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", construct_yaml_str)
with open(self.config_file, "r") as stream:
LOG.debug("Opening config file to retrieve settings.")
yaml_settings = yaml.safe_load(stream)
pretty_settings = yaml.dump(yaml_settings, width=1, indent=4)
LOG.debug("Settings retrieved from user config file: %s", pretty_settings)
if yaml_settings is not None:
# Update self.settings, but only currently valid settings.
for name, value in yaml_settings.items():
if name == "subscriptions":
pass
elif name not in self.settings:
LOG.debug("Setting %s is not a valid setting, ignoring.", name)
else:
self.settings[name] = value
fail_count = 0
for i, yaml_sub in enumerate(yaml_settings.get("subscriptions", [])):
sub = Subscription.Subscription.parse_from_user_yaml(yaml_sub, self.settings)
if sub is None:
LOG.debug("Unable to parse user YAML for sub # %s - something is wrong.",
i + 1)
fail_count += 1
continue
self.subscriptions.append(sub)
if fail_count > 0:
LOG.error("Some subscriptions from config file couldn't be parsed - check logs.")
return True
def addOrderedDictToYamlInterpreter():
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
return dumper.represent_dict(data.iteritems())
def dict_constructor(loader, node):
return OrderedDict(loader.construct_pairs(node))
SafeDumper.add_representer(OrderedDict, dict_representer)
SafeLoader.add_constructor(_mapping_tag, dict_constructor)
SafeDumper.add_representer(str, SafeRepresenter.represent_str)
def yaml_load_unicode(stream):
import yaml
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
return yaml.load(stream)
def fix_yaml_loader():
"""确保读出的yaml文件内容
可以被unicode编码
"""
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def getSchema():
def construct_yaml_str(self, node):
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SCHEMA_DIR = getattr(settings, "STATIC_ROOT", None)
SCHEMA_FILE = SCHEMA_DIR + 'schema.yaml'
stream = open(SCHEMA_FILE, 'r')
schema = yaml.load(stream)
return schema
def __init__(self, filename):
# Make sure pyyaml always returns unicode
def construct_yaml_str(self, node):
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str',
construct_yaml_str)
with open(filename, 'r') as f:
try:
self.data = yaml.safe_load(f)
except yaml.YAMLError as e:
raise e
def _handle_quirks(self):
if self._unicode_quirk:
self._logger.debug('Enabling unicode quirk')
def construct_yaml_str(self, node):
try:
rawdata = b''.join([chr(ord(x)) for x in self.construct_scalar(node)])
return rawdata.decode('utf8')
except ValueError:
# apparently sometimes the data is already correctly encoded
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def _handle_quirks(self):
if self._unicode_quirk:
self._logger.debug('Enabling unicode quirk')
def construct_yaml_str(self, node):
try:
rawdata = b''.join([chr(ord(x)) for x in self.construct_scalar(node)])
return rawdata.decode('utf8')
except ValueError:
# apparently sometimes the data is already correctly encoded
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def regex_constructor(loader: yaml.SafeLoader,
node: yaml.nodes.Node) -> re.Pattern:
'''
Returns the regex pattern.
'''
value = loader.construct_scalar(node)
return re.compile(value)
def __init__(self, context, stream, iter_entry_points=iter_entry_points):
self.context = context
for point in list(iter_entry_points(self.EP_GROUP)):
try:
directive = point.load()
if point.name.startswith('tag:'):
directive_name = point.name
else:
directive_name = '!' + point.name
self.add_constructor(directive_name, wrap_directive(directive))
except ImportError:
logging.info('Could not import repoze.configuration.directive '
'entry point "%s"' % point)
self.add_constructor('tag:yaml.org,2002:str', self.interpolate_str)
SafeLoader.__init__(self, stream)
while self.check_data():
self.get_data()
def _do_loads(self, s, *args, **kwargs):
import yaml
from yaml import Loader, SafeLoader
# Force Unicode string output according to this SO question
# 2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
_STR_TAG = 'tag:yaml.org,2002:str'
Loader.add_constructor(_STR_TAG, construct_yaml_str)
SafeLoader.add_constructor(_STR_TAG, construct_yaml_str)
# TODO: optionally utilize C acceleration if available
return yaml.load(s, *args, **kwargs)
def get_config_yaml( path_config=os.path.dirname(os.path.realpath(__file__)) + "/../config.d/config.yaml", name_config="openstack"):
import yaml
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
f = open(path_config)
# use safe_load instead load
dataMap = yaml.load(f)[name_config]
f.close()
return dataMap
def default_mapping_constructor(loader: yaml.SafeLoader,
node: yaml.nodes.MappingNode,
default_map: dict) -> dict:
return recursive_dict_update(
default_map,
loader.construct_mapping(node, True),
copy=True,
)
def insert_constructor(loader: SafeLoader, node: Node) -> list:
if isinstance(node.value[0], ScalarNode):
info = InsertInfo(
sequence=loader.construct_object(node.value[0], deep=True))
else:
info = InsertInfo(
*loader.construct_sequence(node.value[0], deep=True))
current_list: List[Any] = info.sequence # already constructed
input_list: List[Any] = [
loader.construct_object(n, deep=True) for n in node.value[1:]
]
if info.replace_format is None and info.positions is None:
return current_list + input_list
def item_id(item: Any, idx: int) -> int:
return idx if info.replace_format is None else formatter.format(
info.replace_format, l=item)
# ordered
result_dict = {
item_id(item, idx): item
for idx, item in enumerate(current_list)
}
input_dict = {
item_id(item, idx + len(result_dict)): item
for idx, item in enumerate(input_list)
}
to_pos = {}
to_end = []
for input_pos, (input_id,
input_item) in zip_longest(info.positions or [],
input_dict.items()):
if input_pos is not None:
result_dict.pop(input_id, None)
to_pos[input_pos] = input_item
elif input_id in result_dict:
result_dict[input_id] = input_item
else:
to_end.append(input_item)
result_list = list(result_dict.values()) + to_end
for item in sorted(to_pos):
result_list.insert(item, to_pos[item])
return result_list
def __call__(self, loader: SafeLoader, node: Node) -> str:
""" Implement the tag constructor interface.
:param loader: YAML loader
:param node: YAML node to process
:return: final value
"""
value = loader.construct_scalar(node)
return Template(value).substitute(self._params)
def __init__(self, context, stream, iter_entry_points=iter_entry_points):
self.context = context
for point in list(iter_entry_points(self.EP_GROUP)):
try:
directive = point.load()
if point.name.startswith('tag:'):
directive_name = point.name
else:
directive_name = '!' + point.name
self.add_constructor(directive_name, wrap_directive(directive))
except ImportError:
logging.info(
'Could not import repoze.configuration.directive '
'entry point "%s"' % point)
self.add_constructor('tag:yaml.org,2002:str', self.interpolate_str)
SafeLoader.__init__(self, stream)
while self.check_data():
self.get_data()
def each_constructor(loader: SafeLoader, node: Node) -> list:
input_list, attr, is_required = loader.construct_sequence(node, deep=True)
result = []
for item in input_list:
try:
result.append(attrgetter(attr)(attr_wrap(item)))
except KeyError:
if is_required:
raise
return result
def join_constructor(loader: SafeLoader, node: Node) -> str:
info = loader.construct_sequence(node, deep=True)
separator, input_list = info[0], info[1]
format = info[2] if len(info) == 3 else '{l}'
def flatten(l: list) -> list:
return sum(map(flatten, l), []) if isinstance(l, list) else [l]
input_list = flatten(input_list)
return separator.join(value for item in input_list
if (value := formatter.format(format, l=item)))
def _stats_config_constructor(
loader: yaml.SafeLoader,
node: yaml.nodes.ScalarNode) -> StatsStorageConfig:
"""Returns a scalar constructor that instructs PyYAML how to deserialize a StatsStorageConfig
instance from an "arg string" in the format of "<STORAGE_TYPE>:<RUNNING_ENVIRONMENT>:<TAG>:<PATH_OR_BUCKET>".
Args:
loader (yaml.SafeLoader): PyYAML's default SafeLoader instance
node (yaml.nodes.ScalarNode): A YAML scalar node with a string value representing a StatsStorageConfing instance
Returns:
StatsStorageConfig: A StatsStorageConfig instance deserialized from its string scalar representation
"""
return StatsStorageConfig.from_arg_str(loader.construct_scalar(node))
def __load_checksums(self):
checksum_data = self.kv.get('{0}/containers/checksums'.format(
self.name))
try:
if checksum_data:
data = normalize_keys(SafeLoader(checksum_data).get_data(),
snake_case=False)
else:
data = {}
return data
except YAMLError as e:
if hasattr(e, 'problem_mark'):
raise SyntaxError(
"There is a syntax error in the rules line: {0} column: {1}"
.format(e.problem_mark.line, e.problem_mark.column))
else:
raise SyntaxError("There is a syntax error in the rules.")
import os
from path import path
# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
"""
Override the default string handling function
to always return unicode objects
"""
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
# SERVICE_VARIANT specifies name of the variant used, which decides what YAML
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
# CONFIG_ROOT specifies the directory where the YAML configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))
# CONFIG_PREFIX specifies the prefix of the YAML configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
def parent_constructor(loader: SafeLoader, node: Node):
result = attrgetter(loader.construct_scalar(node))(_data)
return result
def get_constructor(loader: SafeLoader, node: Node):
input_list, attr = loader.construct_sequence(node, deep=True)
return attrgetter(attr)(attr_wrap(input_list))
def concat_constructor(loader: SafeLoader, node: Node) -> list:
input_list: List[list] = loader.construct_sequence(node, deep=True)
result = [item for list_ in input_list for item in list_]
return result
def merge_constructor(loader: SafeLoader, node: Node) -> dict:
input_dict: dict = loader.construct_mapping(node, deep=True)
base_dict = input_dict.pop('<')
merged = {**base_dict, **input_dict}
return merged
The files must be either in ``INI`` format or in ``YAML`` format, in
which case, it must end in ``.yaml`` or ``.yml``.
"""
import sys
import os
import configparser
from logging.config import fileConfig, dictConfig
from pathlib import Path
import yaml
from yaml import SafeLoader as sf
_here = Path(__file__).parent
_config_files = [_here / 'defaults.ini', '/etc/ega/conf.ini']
sf.add_constructor('tag:yaml.org,2002:python/tuple',
lambda self, node: tuple(sf.construct_sequence(self, node)))
class Configuration(configparser.ConfigParser):
"""Configuration from config_files or environment variables or config server (e.g. Spring Cloud Config)."""
log_conf = None
def _load_conf(self, args=None, encoding='utf-8'):
"""Load a configuration file from `args`."""
# Finding the --conf file
try:
conf_file = Path(args[args.index('--conf') + 1]).expanduser()
if conf_file not in _config_files:
_config_files.append(conf_file)
print(f"Overriding configuration settings with {conf_file}",
def from_yaml(cls: Type['VerediYamlDocument'], loader: yaml.SafeLoader,
node: yaml.Node):
return loader.construct_yaml_object(node, cls)
def addNobodyKnewResultToYamlInterpreter():
SafeDumper.add_representer(
NobodyKnewResult,
lambda dumper, y: dumper.represent_scalar("!nobody", ""))
SafeLoader.add_constructor("!nobody", lambda x, y: NobodyKnewResult())
from .game import GameStateModel, NobodyKnewResult from .loader import GameStateLoader, SpecialField from yaml import SafeLoader, SafeDumper SafeLoader.add_constructor(u'!double', SpecialField.doubleJeopardyConstructor) SafeLoader.add_constructor(u'!image', SpecialField.imageAnswerConstructor) doubleAndImageConstructor = SpecialField.makeDoubleJeopardyAndConstructor(SpecialField.imageAnswerConstructor) SafeLoader.add_constructor(u'!double*image', doubleAndImageConstructor)
node=node, value=new_value)
raise YAMLError(msg)
return new_value
def _constructor_envfile_variables(loader, node):
"""
Extracts the environment variable from the node's value.
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: value read from file pointed to by environment variable
"""
raw_value = loader.construct_scalar(node)
filepath = os.environ.get(raw_value)
try:
with open(filepath, "r") as fd:
new_value = fd.read()
except (TypeError, IOError) as e:
msg = "Cannot construct value from {node}: {path}".format(
node=node, path=filepath)
raise YAMLError(msg) from e
else:
return new_value
TAG_ENV = "!ENV"
TAG_ENVFILE = "!ENVFILE"
_safe_loader.add_constructor(TAG_ENV, _constructor_env_variables)
_safe_loader.add_constructor(TAG_ENVFILE, _constructor_envfile_variables)
from path import Path as path
# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
"""
Override the default string handling function
to always return unicode objects
"""
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
# SERVICE_VARIANT specifies name of the variant used, which decides what YAML
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
# CONFIG_ROOT specifies the directory where the YAML configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))
# CONFIG_PREFIX specifies the prefix of the YAML configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
from yaml import safe_load, safe_dump, SafeLoader
SafeLoader.add_constructor('tag:yaml.org,2002:python/unicode', SafeLoader.construct_yaml_str)
def yaml_load(stream):
"""Loads a dictionary from a stream"""
return safe_load(stream)
def yaml_dump(data, stream=None):
"""Dumps an object to a YAML string"""
return safe_dump(data, stream=stream, default_flow_style=False)
