def __load(self, rules_data: str) -> None: """ loads a yaml str, serializes and normalized the rule data. Then assigns the rule data to properties. :param config_file: A :string: loaded from a yaml file. """ if not isinstance(rules_data, str): raise TypeError('rules_data must be a str.') try: data = SafeLoader(rules_data).get_data() if data is None: raise AttributeError('The rules must have data in it.') data = normalize_keys(data, snake_case=False) for key, value in data.items(): variable_name = '_{0}'.format(key) if hasattr(self, variable_name): setattr(self, variable_name, value) else: raise AttributeError('{0} isn\'t a valid rule attribute.'.format(key)) except YAMLError as e: if hasattr(e, 'problem_mark'): raise SyntaxError( "There is a syntax error in the rules line: {0} column: {1}".format( e.problem_mark.line, e.problem_mark.column ) ) else: raise SyntaxError("There is a syntax error in the rules.")
def __load(self, rules_data: str) -> None: """ loads a yaml str, serializes and normalized the rule data. Then assigns the rule data to properties. :param config_file: A :string: loaded from a yaml file. """ if not isinstance(rules_data, str): raise TypeError('rules_data must be a str.') try: data = SafeLoader(rules_data).get_data() if data is None: raise AttributeError('The rules must have data in it.') data = normalize_keys(data, snake_case=False) for key, value in data.items(): variable_name = '_{0}'.format(key) if hasattr(self, variable_name): setattr(self, variable_name, value) else: raise AttributeError( '{0} isn\'t a valid rule attribute.'.format(key)) except YAMLError as e: if hasattr(e, 'problem_mark'): raise SyntaxError( "There is a syntax error in the rules line: {0} column: {1}" .format(e.problem_mark.line, e.problem_mark.column)) else: raise SyntaxError("There is a syntax error in the rules.")
def __init__(self, stream): SafeLoader.__init__(self, stream) self.library = Library(self) self.represent = representer.Representer().represent_data if Loader.anchors is not None: self.set_anchors(Loader.anchors) if Loader.filepath is None: self.importpath = os.getcwd() else: self.importpath = os.path.abspath(os.path.dirname(Loader.filepath))
def fix_yaml_loader(): """Ensure that any string read by yaml is represented as unicode.""" from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def _load_user_settings(self): """Load user settings from config file.""" successful = _ensure_file(self.config_file) if not successful: LOG.error("Unable to load user config file.") return self.subscriptions = [] # Python 2/PyPy shim, per # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects # Override the default string handling function to always return unicode objects. def construct_yaml_str(self, node): """Override to force PyYAML to handle unicode on Python 2.""" return self.construct_scalar(node) SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", construct_yaml_str) with open(self.config_file, "r") as stream: LOG.debug("Opening config file to retrieve settings.") yaml_settings = yaml.safe_load(stream) pretty_settings = yaml.dump(yaml_settings, width=1, indent=4) LOG.debug("Settings retrieved from user config file: %s", pretty_settings) if yaml_settings is not None: # Update self.settings, but only currently valid settings. for name, value in yaml_settings.items(): if name == "subscriptions": pass elif name not in self.settings: LOG.debug("Setting %s is not a valid setting, ignoring.", name) else: self.settings[name] = value fail_count = 0 for i, yaml_sub in enumerate(yaml_settings.get("subscriptions", [])): sub = Subscription.Subscription.parse_from_user_yaml(yaml_sub, self.settings) if sub is None: LOG.debug("Unable to parse user YAML for sub # %s - something is wrong.", i + 1) fail_count += 1 continue self.subscriptions.append(sub) if fail_count > 0: LOG.error("Some subscriptions from config file couldn't be parsed - check logs.") return True
def addOrderedDictToYamlInterpreter(): _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG def dict_representer(dumper, data): return dumper.represent_dict(data.iteritems()) def dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) SafeDumper.add_representer(OrderedDict, dict_representer) SafeLoader.add_constructor(_mapping_tag, dict_constructor) SafeDumper.add_representer(str, SafeRepresenter.represent_str)
def yaml_load_unicode(stream): import yaml from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) return yaml.load(stream)
def fix_yaml_loader(): """确保读出的yaml文件内容 可以被unicode编码 """ from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def getSchema(): def construct_yaml_str(self, node): return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SCHEMA_DIR = getattr(settings, "STATIC_ROOT", None) SCHEMA_FILE = SCHEMA_DIR + 'schema.yaml' stream = open(SCHEMA_FILE, 'r') schema = yaml.load(stream) return schema
def __init__(self, filename): # Make sure pyyaml always returns unicode def construct_yaml_str(self, node): return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) with open(filename, 'r') as f: try: self.data = yaml.safe_load(f) except yaml.YAMLError as e: raise e
def _handle_quirks(self): if self._unicode_quirk: self._logger.debug('Enabling unicode quirk') def construct_yaml_str(self, node): try: rawdata = b''.join([chr(ord(x)) for x in self.construct_scalar(node)]) return rawdata.decode('utf8') except ValueError: # apparently sometimes the data is already correctly encoded return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def regex_constructor(loader: yaml.SafeLoader, node: yaml.nodes.Node) -> re.Pattern: ''' Returns the regex pattern. ''' value = loader.construct_scalar(node) return re.compile(value)
def __init__(self, context, stream, iter_entry_points=iter_entry_points): self.context = context for point in list(iter_entry_points(self.EP_GROUP)): try: directive = point.load() if point.name.startswith('tag:'): directive_name = point.name else: directive_name = '!' + point.name self.add_constructor(directive_name, wrap_directive(directive)) except ImportError: logging.info('Could not import repoze.configuration.directive ' 'entry point "%s"' % point) self.add_constructor('tag:yaml.org,2002:str', self.interpolate_str) SafeLoader.__init__(self, stream) while self.check_data(): self.get_data()
def _do_loads(self, s, *args, **kwargs): import yaml from yaml import Loader, SafeLoader # Force Unicode string output according to this SO question # 2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) _STR_TAG = 'tag:yaml.org,2002:str' Loader.add_constructor(_STR_TAG, construct_yaml_str) SafeLoader.add_constructor(_STR_TAG, construct_yaml_str) # TODO: optionally utilize C acceleration if available return yaml.load(s, *args, **kwargs)
def get_config_yaml( path_config=os.path.dirname(os.path.realpath(__file__)) + "/../config.d/config.yaml", name_config="openstack"): import yaml from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) f = open(path_config) # use safe_load instead load dataMap = yaml.load(f)[name_config] f.close() return dataMap
def default_mapping_constructor(loader: yaml.SafeLoader, node: yaml.nodes.MappingNode, default_map: dict) -> dict: return recursive_dict_update( default_map, loader.construct_mapping(node, True), copy=True, )
def insert_constructor(loader: SafeLoader, node: Node) -> list: if isinstance(node.value[0], ScalarNode): info = InsertInfo( sequence=loader.construct_object(node.value[0], deep=True)) else: info = InsertInfo( *loader.construct_sequence(node.value[0], deep=True)) current_list: List[Any] = info.sequence # already constructed input_list: List[Any] = [ loader.construct_object(n, deep=True) for n in node.value[1:] ] if info.replace_format is None and info.positions is None: return current_list + input_list def item_id(item: Any, idx: int) -> int: return idx if info.replace_format is None else formatter.format( info.replace_format, l=item) # ordered result_dict = { item_id(item, idx): item for idx, item in enumerate(current_list) } input_dict = { item_id(item, idx + len(result_dict)): item for idx, item in enumerate(input_list) } to_pos = {} to_end = [] for input_pos, (input_id, input_item) in zip_longest(info.positions or [], input_dict.items()): if input_pos is not None: result_dict.pop(input_id, None) to_pos[input_pos] = input_item elif input_id in result_dict: result_dict[input_id] = input_item else: to_end.append(input_item) result_list = list(result_dict.values()) + to_end for item in sorted(to_pos): result_list.insert(item, to_pos[item]) return result_list
def __call__(self, loader: SafeLoader, node: Node) -> str: """ Implement the tag constructor interface. :param loader: YAML loader :param node: YAML node to process :return: final value """ value = loader.construct_scalar(node) return Template(value).substitute(self._params)
def __init__(self, context, stream, iter_entry_points=iter_entry_points): self.context = context for point in list(iter_entry_points(self.EP_GROUP)): try: directive = point.load() if point.name.startswith('tag:'): directive_name = point.name else: directive_name = '!' + point.name self.add_constructor(directive_name, wrap_directive(directive)) except ImportError: logging.info( 'Could not import repoze.configuration.directive ' 'entry point "%s"' % point) self.add_constructor('tag:yaml.org,2002:str', self.interpolate_str) SafeLoader.__init__(self, stream) while self.check_data(): self.get_data()
def each_constructor(loader: SafeLoader, node: Node) -> list: input_list, attr, is_required = loader.construct_sequence(node, deep=True) result = [] for item in input_list: try: result.append(attrgetter(attr)(attr_wrap(item))) except KeyError: if is_required: raise return result
def join_constructor(loader: SafeLoader, node: Node) -> str: info = loader.construct_sequence(node, deep=True) separator, input_list = info[0], info[1] format = info[2] if len(info) == 3 else '{l}' def flatten(l: list) -> list: return sum(map(flatten, l), []) if isinstance(l, list) else [l] input_list = flatten(input_list) return separator.join(value for item in input_list if (value := formatter.format(format, l=item)))
def _stats_config_constructor( loader: yaml.SafeLoader, node: yaml.nodes.ScalarNode) -> StatsStorageConfig: """Returns a scalar constructor that instructs PyYAML how to deserialize a StatsStorageConfig instance from an "arg string" in the format of "<STORAGE_TYPE>:<RUNNING_ENVIRONMENT>:<TAG>:<PATH_OR_BUCKET>". Args: loader (yaml.SafeLoader): PyYAML's default SafeLoader instance node (yaml.nodes.ScalarNode): A YAML scalar node with a string value representing a StatsStorageConfing instance Returns: StatsStorageConfig: A StatsStorageConfig instance deserialized from its string scalar representation """ return StatsStorageConfig.from_arg_str(loader.construct_scalar(node))
def __load_checksums(self): checksum_data = self.kv.get('{0}/containers/checksums'.format( self.name)) try: if checksum_data: data = normalize_keys(SafeLoader(checksum_data).get_data(), snake_case=False) else: data = {} return data except YAMLError as e: if hasattr(e, 'problem_mark'): raise SyntaxError( "There is a syntax error in the rules line: {0} column: {1}" .format(e.problem_mark.line, e.problem_mark.column)) else: raise SyntaxError("There is a syntax error in the rules.")
import os from path import path # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader def construct_yaml_str(self, node): """ Override the default string handling function to always return unicode objects """ return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) # SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) # CONFIG_ROOT specifies the directory where the YAML configuration # files are expected to be found. If not specified, use the project # directory. CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) # CONFIG_PREFIX specifies the prefix of the YAML configuration files, # based on the service variant. If no variant is use, don't use a # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
def parent_constructor(loader: SafeLoader, node: Node): result = attrgetter(loader.construct_scalar(node))(_data) return result
def get_constructor(loader: SafeLoader, node: Node): input_list, attr = loader.construct_sequence(node, deep=True) return attrgetter(attr)(attr_wrap(input_list))
def concat_constructor(loader: SafeLoader, node: Node) -> list: input_list: List[list] = loader.construct_sequence(node, deep=True) result = [item for list_ in input_list for item in list_] return result
def merge_constructor(loader: SafeLoader, node: Node) -> dict: input_dict: dict = loader.construct_mapping(node, deep=True) base_dict = input_dict.pop('<') merged = {**base_dict, **input_dict} return merged
The files must be either in ``INI`` format or in ``YAML`` format, in which case, it must end in ``.yaml`` or ``.yml``. """ import sys import os import configparser from logging.config import fileConfig, dictConfig from pathlib import Path import yaml from yaml import SafeLoader as sf _here = Path(__file__).parent _config_files = [_here / 'defaults.ini', '/etc/ega/conf.ini'] sf.add_constructor('tag:yaml.org,2002:python/tuple', lambda self, node: tuple(sf.construct_sequence(self, node))) class Configuration(configparser.ConfigParser): """Configuration from config_files or environment variables or config server (e.g. Spring Cloud Config).""" log_conf = None def _load_conf(self, args=None, encoding='utf-8'): """Load a configuration file from `args`.""" # Finding the --conf file try: conf_file = Path(args[args.index('--conf') + 1]).expanduser() if conf_file not in _config_files: _config_files.append(conf_file) print(f"Overriding configuration settings with {conf_file}",
def from_yaml(cls: Type['VerediYamlDocument'], loader: yaml.SafeLoader, node: yaml.Node): return loader.construct_yaml_object(node, cls)
def addNobodyKnewResultToYamlInterpreter(): SafeDumper.add_representer( NobodyKnewResult, lambda dumper, y: dumper.represent_scalar("!nobody", "")) SafeLoader.add_constructor("!nobody", lambda x, y: NobodyKnewResult())
from .game import GameStateModel, NobodyKnewResult from .loader import GameStateLoader, SpecialField from yaml import SafeLoader, SafeDumper SafeLoader.add_constructor(u'!double', SpecialField.doubleJeopardyConstructor) SafeLoader.add_constructor(u'!image', SpecialField.imageAnswerConstructor) doubleAndImageConstructor = SpecialField.makeDoubleJeopardyAndConstructor(SpecialField.imageAnswerConstructor) SafeLoader.add_constructor(u'!double*image', doubleAndImageConstructor)
node=node, value=new_value) raise YAMLError(msg) return new_value def _constructor_envfile_variables(loader, node): """ Extracts the environment variable from the node's value. :param yaml.Loader loader: the yaml loader :param node: the current node in the yaml :return: value read from file pointed to by environment variable """ raw_value = loader.construct_scalar(node) filepath = os.environ.get(raw_value) try: with open(filepath, "r") as fd: new_value = fd.read() except (TypeError, IOError) as e: msg = "Cannot construct value from {node}: {path}".format( node=node, path=filepath) raise YAMLError(msg) from e else: return new_value TAG_ENV = "!ENV" TAG_ENVFILE = "!ENVFILE" _safe_loader.add_constructor(TAG_ENV, _constructor_env_variables) _safe_loader.add_constructor(TAG_ENVFILE, _constructor_envfile_variables)
from path import Path as path # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader def construct_yaml_str(self, node): """ Override the default string handling function to always return unicode objects """ return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) # SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) # CONFIG_ROOT specifies the directory where the YAML configuration # files are expected to be found. If not specified, use the project # directory. CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) # CONFIG_PREFIX specifies the prefix of the YAML configuration files, # based on the service variant. If no variant is use, don't use a # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals from yaml import safe_load, safe_dump, SafeLoader SafeLoader.add_constructor('tag:yaml.org,2002:python/unicode', SafeLoader.construct_yaml_str) def yaml_load(stream): """Loads a dictionary from a stream""" return safe_load(stream) def yaml_dump(data, stream=None): """Dumps an object to a YAML string""" return safe_dump(data, stream=stream, default_flow_style=False)