def fix_yaml_loader(): """Ensure that any string read by yaml is represented as unicode.""" from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def _handle_quirks(self): if self._unicode_quirk: self._logger.debug('Enabling unicode quirk') def construct_yaml_str(self, node): try: rawdata = b''.join([chr(ord(x)) for x in self.construct_scalar(node)]) return rawdata.decode('utf8') except ValueError: # apparently sometimes the data is already correctly encoded return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def get_config_yaml( path_config=os.path.dirname(os.path.realpath(__file__)) + "/../config.d/config.yaml", name_config="openstack"): import yaml from yaml import Loader, SafeLoader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) f = open(path_config) # use safe_load instead load dataMap = yaml.load(f)[name_config] f.close() return dataMap
def _do_loads(self, s, *args, **kwargs): import yaml from yaml import Loader, SafeLoader # Force Unicode string output according to this SO question # 2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) _STR_TAG = 'tag:yaml.org,2002:str' Loader.add_constructor(_STR_TAG, construct_yaml_str) SafeLoader.add_constructor(_STR_TAG, construct_yaml_str) # TODO: optionally utilize C acceleration if available return yaml.load(s, *args, **kwargs)
def load(cls, stream, constructors=None, multi_constructors=None): loader = Loader(stream) cs = dict(cls._constructors) if constructors: cs.update(constructors) mcs = dict(cls._multi_constructors) if multi_constructors: mcs.update(multi_constructors) if cs: for name, constructor in cs.items(): loader.add_constructor(name, constructor) if mcs: for name, constructor in mcs.items(): loader.add_multi_constructor(name, constructor) try: return loader.get_single_data() finally: loader.dispose()
def load(stream, constructors=None): loader = Loader(stream) constructors = constructors or {} if not "timedelta" in constructors: loader.add_constructor("!timedelta", _timedelta_contructor) if not "re" in constructors: loader.add_constructor("!re", _re_constructor) loader.add_multi_constructor("!ref:", _ref_constructor) loader.add_multi_constructor("!factory:", _factory_constructor) loader.add_multi_constructor("!obj:", _obj_constructor) loader.add_multi_constructor("!extends:", _extends_constructor) loader.add_multi_constructor("!include:", _extends_constructor) if constructors: for name, constructor in constructors.items(): loader.add_constructor("!" + name, constructor) try: return loader.get_single_data() finally: loader.dispose()
from util.config_parse import convert_tokens import os from path import path # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader def construct_yaml_str(self, node): """ Override the default string handling function to always return unicode objects """ return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) # SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) # CONFIG_ROOT specifies the directory where the YAML configuration # files are expected to be found. If not specified, use the project # directory. CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) # CONFIG_PREFIX specifies the prefix of the YAML configuration files, # based on the service variant. If no variant is use, don't use a # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
action='store_true') args = parser.parse_args() try: import yaml except ImportError: raise Exception('You need to install pyaml to run the tests') # from http://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader def construct_yaml_str(self, node): return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) try: from bs4 import BeautifulSoup except ImportError: try: from BeautifulSoup import BeautifulSoup except ImportError: raise Exception('You need to install BeautifulSoup to run the tests') def success(text): return (u'\033[92m%s\033[0m\n' % text).encode('utf-8')
def sqlalchemy_constructor(loader, node): data = loader.construct_sequence(node) if data[1] is None: # object was never persisted return None return getattr(db.m, data[0]).get(data[1]) def zodb_constructor(loader, node): oid = loader.construct_sequence(node)[0] return loader.zodb_root._p_jar[oid] Dumper.add_multi_representer(db.Model, sqlalchemy_representer) Dumper.add_multi_representer(Persistent, zodb_representer) Loader.add_constructor('!sqlalchemy', sqlalchemy_constructor) Loader.add_constructor('!zodb', zodb_constructor) STORE_MAP = {'setdict': lambda: defaultdict(set)} class SharedNamespace(object): def __init__(self, name, zodb_root, store_types): self.name = name self._store_types = store_types self._stores = { k: STORE_MAP.get(ktype, ktype)() for k, ktype in store_types.viewitems() } def __getattr__(self, key):
def dict_representer(dumper, data): return dumper.represent_dict(data.items()) def dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) def represent_none(self, data): return self.represent_scalar(u'tag:yaml.org,2002:null', u'') Dumper.add_representer(OrderedDict, dict_representer) Dumper.add_representer(type(None), represent_none) Loader.add_constructor(_mapping_tag, dict_constructor) Dumper.add_representer(str, SafeRepresenter.represent_str) Dumper.add_representer(dict, SafeRepresenter.represent_dict) # Dumper.add_representer(unicode, SafeRepresenter.represent_unicode) # Utility def phex(p, llen=0): if isinstance(p, str): pp = p.encode() else:
""" Class description goes here. """ from uuid import UUID from yaml import Loader, Dumper JAVA_UUID_TAG = u'tag:yaml.org,2002:java.util.UUID' DATACLAY_ID_PREFIX = u'tag:yaml.org,2002:es.bsc.dataclay.util.ids' def uuid_representer(dumper, data): return dumper.represent_scalar(JAVA_UUID_TAG, str(data)) def uuid_constructor(loader, node): value = loader.construct_scalar(node) return UUID(value) Dumper.add_representer(UUID, uuid_representer) Loader.add_constructor(JAVA_UUID_TAG, uuid_constructor) Loader.add_multi_constructor( DATACLAY_ID_PREFIX, # This ignores the tag, as ImplementationID, OperationID, *ID are always # used directly as their UUID, not their specific type. lambda loader, _, node: uuid_constructor(loader, node))
from future.utils import iteritems, itervalues from snips_nlu_utils import normalize from yaml import Loader, SafeLoader from snips_nlu.constants import ( DATA, ENTITIES, ENTITY, INTENTS, TEXT, UTTERANCES) from snips_nlu.entity_parser.builtin_entity_parser import is_gazetteer_entity def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor("tag:yaml.org,2002:str", construct_yaml_str) SafeLoader.add_constructor("tag:yaml.org,2002:str", construct_yaml_str) def extract_utterance_entities(dataset): entities_values = {ent_name: set() for ent_name in dataset[ENTITIES]} for intent in itervalues(dataset[INTENTS]): for utterance in intent[UTTERANCES]: for chunk in utterance[DATA]: if ENTITY in chunk: entities_values[chunk[ENTITY]].add(chunk[TEXT].strip()) return {k: list(v) for k, v in iteritems(entities_values)} def extract_intent_entities(dataset, entity_filter=None):
def lonely_equal_constructor(loader, node): """Solve/monkey-patch a very old bug. https://bitbucket.org/xi/pyyaml/issues/49/plain-equal-sign-as-node-content-results """ s = loader.construct_scalar(node) return s def lang_constructor(loader, node): """Language is parsed as a GRPC enum.""" s = loader.construct_scalar(node) return Langs.Value(s) Loader.add_constructor(u"tag:yaml.org,2002:value", lonely_equal_constructor) # The tuple is a bit special itself Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.util.structs.Tuple", tuple_constructor) # Not needed for Python, but nice to avoid errors Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.util.management.classmgr.features.Feature$FeatureType", feature_constructor) # The language is very special Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.communication.grpc.messages.common.CommonMessages$Langs", lang_constructor) for prefix in IGNORE_PREFIXES: yaml_tag_prefix = u"tag:yaml.org,2002:%s" % prefix Loader.add_multi_constructor( yaml_tag_prefix,