def read_yaml(yaml_path): import yaml import re with open(yaml_path, mode='r') as f: try: # use SafeLoader's over Loader to prevent against arbitrary python object execution # Use C loaders if possible, faster from yaml import CSafeLoader as Loader # CLoader except ImportError: from yaml import SafeLoader as Loader # Loader _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG def dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) Loader.add_constructor(_mapping_tag, dict_constructor) # compiled resolver to correctly parse scientific notation numbers Loader.add_implicit_resolver( u'tag:yaml.org,2002:float', re.compile(u'''^(?: [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) |\\.[0-9_]+(?:[eE][-+]?[0-9]+)? |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* |[-+]?\\.(?:inf|Inf|INF) |\\.(?:nan|NaN|NAN))$''', re.X), list(u'-+0123456789.')) return yaml.load(f, Loader=Loader)
def setupYamlLoadersAndDumpers(): try: # Use the native code backends, if available. from yaml import CSafeLoader as Loader, CDumper as Dumper except ImportError: from yaml import SafeLoader as Loader, Dumper def string_representer(dumper, value): style = None # If it has newlines, request a block style. if "\n" in value: style = "|" # if it looks like an identifier, use no style if re.match( r"^[a-zA-Z0-9_\-/]+$", value) and len(value) < 60 and value not in ("true", "false"): style = '' return dumper.represent_scalar(u'tag:yaml.org,2002:str', value, style=style) Dumper.add_representer(str, string_representer) Dumper.add_representer(unicode, string_representer) Dumper.add_representer(int, lambda dumper, value : \ dumper.represent_scalar(u'tag:yaml.org,2002:int', str(value), style='')) Dumper.add_representer(bool, lambda dumper, value : \ dumper.represent_scalar(u'tag:yaml.org,2002:bool', u"true" if value else u"false", style='')) Dumper.add_representer(type(None), lambda dumper, value : \ dumper.represent_scalar(u'tag:yaml.org,2002:null', u"~")) def construct_tuple(loader, node): return tuple(Loader.construct_sequence(loader, node)) Loader.add_constructor(u'tag:yaml.org,2002:seq', construct_tuple)
class _BaseLoader(SafeLoader): """ YAML loader with additional features related to mux """ SafeLoader.add_constructor(u'!include', lambda *_: mux.Control(YAML_INCLUDE)) SafeLoader.add_constructor(u'!using', lambda *_: mux.Control(YAML_USING)) SafeLoader.add_constructor(u'!remove_node', lambda *_: mux.Control(YAML_REMOVE_NODE)) SafeLoader.add_constructor(u'!remove_value', lambda *_: mux.Control(YAML_REMOVE_VALUE)) SafeLoader.add_constructor(u'!filter-only', lambda *_: mux.Control(YAML_FILTER_ONLY)) SafeLoader.add_constructor(u'!filter-out', lambda *_: mux.Control(YAML_FILTER_OUT)) SafeLoader.add_constructor(u'tag:yaml.org,2002:python/dict', SafeLoader.construct_yaml_map) SafeLoader.add_constructor(u'!mux', _mux_loader) SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _mapping_to_tree_loader)
class _BaseLoader(SafeLoader): """ YAML loader with additional features related to mux """ SafeLoader.add_constructor(u'!include', lambda *_: mux.Control(YAML_INCLUDE)) SafeLoader.add_constructor(u'!using', lambda *_: mux.Control(YAML_USING)) SafeLoader.add_constructor(u'!remove_node', lambda *_: mux.Control(YAML_REMOVE_NODE)) SafeLoader.add_constructor(u'!remove_value', lambda *_: mux.Control(YAML_REMOVE_VALUE)) SafeLoader.add_constructor(u'!filter-only', lambda *_: mux.Control(YAML_FILTER_ONLY)) SafeLoader.add_constructor(u'!filter-out', lambda *_: mux.Control(YAML_FILTER_OUT)) SafeLoader.add_constructor( u'tag:yaml.org,2002:python/dict', lambda self, node: dict(self.construct_mapping(node)))
from .constants import (PKG_MANAGER_TO_SYSTEM, JINJA_MACROS_BASE_DEFINITIONS, JINJA_MACROS_HIGHLEVEL_DEFINITIONS) from .constants import DEFAULT_UID_MIN try: from yaml import CSafeLoader as yaml_SafeLoader except ImportError: from yaml import SafeLoader as yaml_SafeLoader def _bool_constructor(self, node): return self.construct_scalar(node) # Don't follow python bool case yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor) def _save_rename(result, stem, prefix): result["{0}_{1}".format(prefix, stem)] = stem def _open_yaml(stream, original_file=None): """ Open given file-like object and parse it as YAML. Optionally, pass the path to the original_file for better error handling when the file contents are passed. Return None if it contains "documentation_complete" key set to "false". """
from horizons.constants import TIER, RES, UNITS, BUILDINGS, PATHS from horizons.util.yamlcachestorage import YamlCacheStorage try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader # make SafeLoader allow unicode def construct_yaml_str(self, node): return self.construct_scalar(node) SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) def parse_token(token, token_klass): """Helper function that tries to parse a constant name. Does not do error detection, but passes unparseable stuff through. Allowed values: integer or token_klass.LIKE_IN_CONSTANTS @param token_klass: "TIER", "RES", "UNITS" or "BUILDINGS" """ classes = { 'TIER': TIER, 'RES': RES, 'UNITS': UNITS, 'BUILDINGS': BUILDINGS }
Ensure :mod:`yaml` does not translate on, yes to True, and off, no to False. .. notes:: In YAML 1.2 the usage of off/on/yes/no was dropped. """ bool_values = { "true": True, "false": False, } v = self.construct_scalar(node) return bool_values.get(v.lower(), v) # Disable deserializing yes/no to boolean True/False _Loader.add_constructor("tag:yaml.org,2002:bool", _construct_bool) # Disable deserializing date, datetime strings to date, datetime object _Loader.add_constructor( "tag:yaml.org,2002:timestamp", _Loader.yaml_constructors["tag:yaml.org,2002:str"], ) # Dumper extras def _represent_path(self, data: Path): """ Serialize a `Path` object to a string. .. warning:: Path("./aaa") serializes to "aaa"
from yaml import CSafeLoader as yaml_SafeLoader except ImportError: from yaml import SafeLoader as yaml_SafeLoader def _bool_constructor(self, node): return self.construct_scalar(node) def _unicode_constructor(self, node): string_like = self.construct_scalar(node) return str(string_like) # Don't follow python bool case yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor) # Python2-relevant - become able to resolve "unicode strings" yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode', _unicode_constructor) class DocumentationNotComplete(Exception): pass def _save_rename(result, stem, prefix): result["{0}_{1}".format(prefix, stem)] = stem def _open_yaml(stream, original_file=None, substitutions_dict={}): """
def construct_odict(load, node): omap = OrderedDict() yield omap if not isinstance(node, yaml.MappingNode): raise yaml.constructor.ConstructorError( "while constructing an ordered map", node.start_mark, "expected a map, but found %s" % node.id, node.start_mark) for key, value in node.value: key = load.construct_object(key) value = load.construct_object(value) omap[key] = value Loader.add_constructor(u'tag:yaml.org,2002:map', construct_odict) def ordered_dict_serializer(self, data): return self.represent_mapping('tag:yaml.org,2002:map', data.items()) Dumper.add_representer(OrderedDict, ordered_dict_serializer) # Likewise, when we store unicode objects make sure we don't write # them with weird YAML tags indicating the Python data type. str-typed # strings come out fine, but unicode strings come out with unnecessary # type tags. The easy solution is this: # # Dumper.add_representer(unicode, lambda dumper, value:
from __future__ import absolute_import from __future__ import unicode_literals from markdown import Extension from markdown.preprocessors import Preprocessor import yaml try: from yaml import CSafeLoader as Loader except ImportError: from yaml import Loader # Override the default string handling function to always return unicode objects def construct_yaml_str(self, node): return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) class MetaYamlExtension (Extension): """Extension for parsing YAML-Metadata with Python-Markdown.""" def extendMarkdown(self, md, md_globals): """Add MetaYamlPreprocessor to Markdown instance.""" md.preprocessors.add("meta_yaml", MetaYamlPreprocessor(md), ">meta") class MetaYamlPreprocessor(Preprocessor): """ Get Meta-Data. A YAML block is delimited by
generate_dois_for_submission from hepdata.modules.records.utils.validators import get_data_validator, get_submission_validator from hepdata.utils.twitter import tweet from invenio_db import db from invenio_pidstore.errors import PIDDoesNotExistError import os from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.exc import SQLAlchemyError import yaml from yaml import CSafeLoader as Loader def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) logging.basicConfig() log = logging.getLogger(__name__) def remove_submission(record_id, version=1): """ Removes the database entries and data files related to a record. :param record_id: :param version: :return: True if Successful, False if the record does not exist. """ hepdata_submissions = HEPSubmission.query.filter_by(
except ImportError: from json import loads else: blogfile_exts.insert(0, "yaml") from functools import partial try: # Use libyaml if present, much faster from yaml import CSafeLoader as _Loader except ImportError: from yaml import SafeLoader as _Loader # Make PyYAML load unicode strings _Loader.add_constructor( 'tag:yaml.org,2002:str', lambda self, node: self.construct_scalar(node) ) loads = partial(_loads, Loader=_Loader) inifile = PIniFile("simpleblog", [ ('source', [ ('encoding', INI_STRING, 'utf-8') ]) ]) def blogdata(data, encoding=inifile.source_encoding): return decode(data, encoding)
return TemplateString(self.construct_scalar(node)) def _construct_yaml_tuple(self, node): # Used to convert sequences from lists to tuples. Only applies to lists # without any nested structures. seq = self.construct_sequence(node) if any(isinstance(e, (list, tuple, dict)) for e in seq): return seq return tuple(seq) def _represent_pipeline_config(self, data): return self.represent_dict(dict(data)) def dump(data, stream=None, Dumper=SafeDumper, **kwargs): """See :py:func:`yaml.dump`.""" return yaml.dump(data, stream=stream, Dumper=Dumper, **kwargs) def load(stream, Loader=SafeLoader): """See :py:func:`yaml.load`.""" return yaml.load(stream, Loader) SafeLoader.add_constructor('tag:yaml.org,2002:seq', _construct_yaml_tuple) SafeDumper.add_representer(phyre_engine.pipeline.PipelineConfig, _represent_pipeline_config)
def construct_odict(load, node): omap = OrderedDict() yield omap if not isinstance(node, yaml.MappingNode): raise yaml.constructor.ConstructorError( "while constructing an ordered map", node.start_mark, "expected a map, but found %s" % node.id, node.start_mark ) for key, value in node.value: key = load.construct_object(key) value = load.construct_object(value) omap[key] = value Loader.add_constructor(u'tag:yaml.org,2002:map', construct_odict) def ordered_dict_serializer(self, data): return self.represent_mapping('tag:yaml.org,2002:map', data.items()) Dumper.add_representer(OrderedDict, ordered_dict_serializer) # Likewise, when we store unicode objects make sure we don't write # them with weird YAML tags indicating the Python data type. The # standard string type is fine. We should do this: # Dumper.add_representer(unicode, lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:str', value)) # # However, the standard PyYAML representer for strings does something # weird: if a value cannot be parsed as an integer quotes are omitted. # # This is incredibly odd when the value is an integer with a leading # zero. These values are typically parsed as octal integers, meaning # quotes would normally be required (that's good). But when the value
JINJA_MACROS_BASE_DEFINITIONS, JINJA_MACROS_HIGHLEVEL_DEFINITIONS) from .constants import DEFAULT_UID_MIN try: from yaml import CSafeLoader as yaml_SafeLoader except ImportError: from yaml import SafeLoader as yaml_SafeLoader def _bool_constructor(self, node): return self.construct_scalar(node) # Don't follow python bool case yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor) def _save_rename(result, stem, prefix): result["{0}_{1}".format(prefix, stem)] = stem def _open_yaml(stream, original_file=None, substitutions_dict={}): """ Open given file-like object and parse it as YAML. Optionally, pass the path to the original_file for better error handling when the file contents are passed. Return None if it contains "documentation_complete" key set to "false". """
import yaml import threading import traceback import logging from horizons.constants import TIER, RES, UNITS, BUILDINGS, PATHS try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader # make SafeLoader allow unicode def construct_yaml_str(self, node): return self.construct_scalar(node) SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) def parse_token(token, token_klass): """Helper function that tries to parse a constant name. Does not do error detection, but passes unparseable stuff through. Allowed values: integer or token_klass.LIKE_IN_CONSTANTS @param token_klass: "TIER", "RES", "UNITS" or "BUILDINGS" """ classes = {'TIER': TIER, 'RES': RES, 'UNITS': UNITS, 'BUILDINGS': BUILDINGS} if isinstance(token, unicode): if token.startswith(token_klass): try: return getattr( classes[token_klass], token.split(".", 2)[1])
from yaml import load as _loads except ImportError: from json import loads else: blogfile_exts.insert(0, "yaml") from functools import partial try: # Use libyaml if present, much faster from yaml import CSafeLoader as _Loader except ImportError: from yaml import SafeLoader as _Loader # Make PyYAML load unicode strings _Loader.add_constructor('tag:yaml.org,2002:str', lambda self, node: self.construct_scalar(node)) loads = partial(_loads, Loader=_Loader) inifile = PIniFile("simpleblog", [('source', [('encoding', INI_STRING, 'utf-8')])]) def blogdata(data, encoding=inifile.source_encoding): return decode(data, encoding) def read_blogfile(filename): with open(filename, 'rb') as f: data = f.read() return blogdata(data)