def fix_yaml_loader():
"""Ensure that any string read by yaml is represented as unicode."""
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def _handle_quirks(self):
if self._unicode_quirk:
self._logger.debug('Enabling unicode quirk')
def construct_yaml_str(self, node):
try:
rawdata = b''.join([chr(ord(x)) for x in self.construct_scalar(node)])
return rawdata.decode('utf8')
except ValueError:
# apparently sometimes the data is already correctly encoded
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def get_config_yaml( path_config=os.path.dirname(os.path.realpath(__file__)) + "/../config.d/config.yaml", name_config="openstack"):
import yaml
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
f = open(path_config)
# use safe_load instead load
dataMap = yaml.load(f)[name_config]
f.close()
return dataMap
def _do_loads(self, s, *args, **kwargs):
import yaml
from yaml import Loader, SafeLoader
# Force Unicode string output according to this SO question
# 2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
_STR_TAG = 'tag:yaml.org,2002:str'
Loader.add_constructor(_STR_TAG, construct_yaml_str)
SafeLoader.add_constructor(_STR_TAG, construct_yaml_str)
# TODO: optionally utilize C acceleration if available
return yaml.load(s, *args, **kwargs)
def load(cls, stream, constructors=None, multi_constructors=None):
loader = Loader(stream)
cs = dict(cls._constructors)
if constructors:
cs.update(constructors)
mcs = dict(cls._multi_constructors)
if multi_constructors:
mcs.update(multi_constructors)
if cs:
for name, constructor in cs.items():
loader.add_constructor(name, constructor)
if mcs:
for name, constructor in mcs.items():
loader.add_multi_constructor(name, constructor)
try:
return loader.get_single_data()
finally:
loader.dispose()
def load(stream, constructors=None):
loader = Loader(stream)
constructors = constructors or {}
if not "timedelta" in constructors:
loader.add_constructor("!timedelta", _timedelta_contructor)
if not "re" in constructors:
loader.add_constructor("!re", _re_constructor)
loader.add_multi_constructor("!ref:", _ref_constructor)
loader.add_multi_constructor("!factory:", _factory_constructor)
loader.add_multi_constructor("!obj:", _obj_constructor)
loader.add_multi_constructor("!extends:", _extends_constructor)
loader.add_multi_constructor("!include:", _extends_constructor)
if constructors:
for name, constructor in constructors.items():
loader.add_constructor("!" + name, constructor)
try:
return loader.get_single_data()
finally:
loader.dispose()
from util.config_parse import convert_tokens
import os
from path import path
# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
"""
Override the default string handling function
to always return unicode objects
"""
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
# SERVICE_VARIANT specifies name of the variant used, which decides what YAML
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
# CONFIG_ROOT specifies the directory where the YAML configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))
# CONFIG_PREFIX specifies the prefix of the YAML configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
action='store_true')
args = parser.parse_args()
try:
import yaml
except ImportError:
raise Exception('You need to install pyaml to run the tests')
# from http://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
from yaml import Loader, SafeLoader
def construct_yaml_str(self, node):
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
try:
from bs4 import BeautifulSoup
except ImportError:
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
raise Exception('You need to install BeautifulSoup to run the tests')
def success(text):
return (u'\033[92m%s\033[0m\n' % text).encode('utf-8')
def sqlalchemy_constructor(loader, node):
data = loader.construct_sequence(node)
if data[1] is None:
# object was never persisted
return None
return getattr(db.m, data[0]).get(data[1])
def zodb_constructor(loader, node):
oid = loader.construct_sequence(node)[0]
return loader.zodb_root._p_jar[oid]
Dumper.add_multi_representer(db.Model, sqlalchemy_representer)
Dumper.add_multi_representer(Persistent, zodb_representer)
Loader.add_constructor('!sqlalchemy', sqlalchemy_constructor)
Loader.add_constructor('!zodb', zodb_constructor)
STORE_MAP = {'setdict': lambda: defaultdict(set)}
class SharedNamespace(object):
def __init__(self, name, zodb_root, store_types):
self.name = name
self._store_types = store_types
self._stores = {
k: STORE_MAP.get(ktype, ktype)()
for k, ktype in store_types.viewitems()
}
def __getattr__(self, key):
def dict_representer(dumper, data): return dumper.represent_dict(data.items()) def dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) def represent_none(self, data): return self.represent_scalar(u'tag:yaml.org,2002:null', u'') Dumper.add_representer(OrderedDict, dict_representer) Dumper.add_representer(type(None), represent_none) Loader.add_constructor(_mapping_tag, dict_constructor) Dumper.add_representer(str, SafeRepresenter.represent_str) Dumper.add_representer(dict, SafeRepresenter.represent_dict) # Dumper.add_representer(unicode, SafeRepresenter.represent_unicode) # Utility def phex(p, llen=0): if isinstance(p, str): pp = p.encode() else:
""" Class description goes here. """
from uuid import UUID
from yaml import Loader, Dumper
JAVA_UUID_TAG = u'tag:yaml.org,2002:java.util.UUID'
DATACLAY_ID_PREFIX = u'tag:yaml.org,2002:es.bsc.dataclay.util.ids'
def uuid_representer(dumper, data):
return dumper.represent_scalar(JAVA_UUID_TAG, str(data))
def uuid_constructor(loader, node):
value = loader.construct_scalar(node)
return UUID(value)
Dumper.add_representer(UUID, uuid_representer)
Loader.add_constructor(JAVA_UUID_TAG, uuid_constructor)
Loader.add_multi_constructor(
DATACLAY_ID_PREFIX,
# This ignores the tag, as ImplementationID, OperationID, *ID are always
# used directly as their UUID, not their specific type.
lambda loader, _, node: uuid_constructor(loader, node))
from future.utils import iteritems, itervalues
from snips_nlu_utils import normalize
from yaml import Loader, SafeLoader
from snips_nlu.constants import (
DATA, ENTITIES, ENTITY, INTENTS, TEXT, UTTERANCES)
from snips_nlu.entity_parser.builtin_entity_parser import is_gazetteer_entity
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor("tag:yaml.org,2002:str", construct_yaml_str)
SafeLoader.add_constructor("tag:yaml.org,2002:str", construct_yaml_str)
def extract_utterance_entities(dataset):
entities_values = {ent_name: set() for ent_name in dataset[ENTITIES]}
for intent in itervalues(dataset[INTENTS]):
for utterance in intent[UTTERANCES]:
for chunk in utterance[DATA]:
if ENTITY in chunk:
entities_values[chunk[ENTITY]].add(chunk[TEXT].strip())
return {k: list(v) for k, v in iteritems(entities_values)}
def extract_intent_entities(dataset, entity_filter=None):
def lonely_equal_constructor(loader, node):
"""Solve/monkey-patch a very old bug.
https://bitbucket.org/xi/pyyaml/issues/49/plain-equal-sign-as-node-content-results
"""
s = loader.construct_scalar(node)
return s
def lang_constructor(loader, node):
"""Language is parsed as a GRPC enum."""
s = loader.construct_scalar(node)
return Langs.Value(s)
Loader.add_constructor(u"tag:yaml.org,2002:value", lonely_equal_constructor)
# The tuple is a bit special itself
Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.util.structs.Tuple", tuple_constructor)
# Not needed for Python, but nice to avoid errors
Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.util.management.classmgr.features.Feature$FeatureType",
feature_constructor)
# The language is very special
Loader.add_constructor(u"tag:yaml.org,2002:es.bsc.dataclay.communication.grpc.messages.common.CommonMessages$Langs", lang_constructor)
for prefix in IGNORE_PREFIXES:
yaml_tag_prefix = u"tag:yaml.org,2002:%s" % prefix
Loader.add_multi_constructor(
yaml_tag_prefix,
