def load_data_definitions(datadir):
"""
Parse the yaml file of base yaml objects and return the information
:arg file yaml_file: Open file object to read the yaml from
:returns: An array of Markets that the user can travel to.
"""
_define_schemas(datadir)
data_file = os.path.join(datadir, 'base', 'stellar-base.yml')
with open(data_file) as f:
loader = Loader(f.read())
base_data = loader.get_single_data()
v_validate(base_data, BASE_SCHEMA)
data_file = os.path.join(datadir, 'base', 'stellar-sol.yml')
with open(data_file) as f:
loader = Loader(f.read())
system_data = loader.get_single_data()
v_validate(system_data, SYSTEM_SCHEMA)
base_data.update(system_data)
del base_data['version']
return base_data
def load_base_types(datadir):
"""
Parse the yaml file of base enum types and return the information
:arg datadir: The data directory to find the types file
:returns: A list of types
"""
flog = mlog.fields(func='load_base_types')
flog.fields(datadir=datadir).debug('Entered load_base_types')
data_file = os.path.join(datadir, 'base', 'stellar-types.yml')
with_file_log = flog.fields(filename=data_file)
with_file_log.debug('constructed data_file path {data_file}', data_file=data_file)
with_file_log.debug('Opening data_file')
with open(data_file, 'r') as data_fh:
with_file_log.debug('reading data_file')
yaml_data = data_fh.read()
with_file_log.fields(yaml=yaml_data).debug('parsing yaml string')
loader = Loader(yaml_data)
data = loader.get_single_data()
flog.fields(data=data).debug('Validating type data structure')
data = v_validate(data, DATA_TYPES_SCHEMA)
flog.debug('Returning type data')
return data
def read_yaml(yaml_path):
import yaml
import re
with open(yaml_path, mode='r') as f:
try:
# use SafeLoader's over Loader to prevent against arbitrary python object execution
# Use C loaders if possible, faster
from yaml import CSafeLoader as Loader # CLoader
except ImportError:
from yaml import SafeLoader as Loader # Loader
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_constructor(loader, node):
return OrderedDict(loader.construct_pairs(node))
Loader.add_constructor(_mapping_tag, dict_constructor)
# compiled resolver to correctly parse scientific notation numbers
Loader.add_implicit_resolver(
u'tag:yaml.org,2002:float',
re.compile(u'''^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|\\.[0-9_]+(?:[eE][-+]?[0-9]+)?
|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
|[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$''', re.X),
list(u'-+0123456789.'))
return yaml.load(f, Loader=Loader)
def setupYamlLoadersAndDumpers():
try:
# Use the native code backends, if available.
from yaml import CSafeLoader as Loader, CDumper as Dumper
except ImportError:
from yaml import SafeLoader as Loader, Dumper
def string_representer(dumper, value):
style = None
# If it has newlines, request a block style.
if "\n" in value:
style = "|"
# if it looks like an identifier, use no style
if re.match(
r"^[a-zA-Z0-9_\-/]+$",
value) and len(value) < 60 and value not in ("true", "false"):
style = ''
return dumper.represent_scalar(u'tag:yaml.org,2002:str',
value,
style=style)
Dumper.add_representer(str, string_representer)
Dumper.add_representer(unicode, string_representer)
Dumper.add_representer(int, lambda dumper, value : \
dumper.represent_scalar(u'tag:yaml.org,2002:int', str(value), style=''))
Dumper.add_representer(bool, lambda dumper, value : \
dumper.represent_scalar(u'tag:yaml.org,2002:bool', u"true" if value else u"false", style=''))
Dumper.add_representer(type(None), lambda dumper, value : \
dumper.represent_scalar(u'tag:yaml.org,2002:null', u"~"))
def construct_tuple(loader, node):
return tuple(Loader.construct_sequence(loader, node))
Loader.add_constructor(u'tag:yaml.org,2002:seq', construct_tuple)
class _BaseLoader(SafeLoader):
"""
YAML loader with additional features related to mux
"""
SafeLoader.add_constructor(u'!include',
lambda *_: mux.Control(YAML_INCLUDE))
SafeLoader.add_constructor(u'!using', lambda *_: mux.Control(YAML_USING))
SafeLoader.add_constructor(u'!remove_node',
lambda *_: mux.Control(YAML_REMOVE_NODE))
SafeLoader.add_constructor(u'!remove_value',
lambda *_: mux.Control(YAML_REMOVE_VALUE))
SafeLoader.add_constructor(u'!filter-only',
lambda *_: mux.Control(YAML_FILTER_ONLY))
SafeLoader.add_constructor(u'!filter-out',
lambda *_: mux.Control(YAML_FILTER_OUT))
SafeLoader.add_constructor(u'tag:yaml.org,2002:python/dict',
SafeLoader.construct_yaml_map)
SafeLoader.add_constructor(u'!mux', _mux_loader)
SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
_mapping_to_tree_loader)
class _BaseLoader(SafeLoader):
"""
YAML loader with additional features related to mux
"""
SafeLoader.add_constructor(u'!include',
lambda *_: mux.Control(YAML_INCLUDE))
SafeLoader.add_constructor(u'!using', lambda *_: mux.Control(YAML_USING))
SafeLoader.add_constructor(u'!remove_node',
lambda *_: mux.Control(YAML_REMOVE_NODE))
SafeLoader.add_constructor(u'!remove_value',
lambda *_: mux.Control(YAML_REMOVE_VALUE))
SafeLoader.add_constructor(u'!filter-only',
lambda *_: mux.Control(YAML_FILTER_ONLY))
SafeLoader.add_constructor(u'!filter-out',
lambda *_: mux.Control(YAML_FILTER_OUT))
SafeLoader.add_constructor(
u'tag:yaml.org,2002:python/dict',
lambda self, node: dict(self.construct_mapping(node)))
generate_dois_for_submission
from hepdata.modules.records.utils.validators import get_data_validator, get_submission_validator
from hepdata.utils.twitter import tweet
from invenio_db import db
from invenio_pidstore.errors import PIDDoesNotExistError
import os
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.exc import SQLAlchemyError
import yaml
from yaml import CSafeLoader as Loader
def construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
logging.basicConfig()
log = logging.getLogger(__name__)
def remove_submission(record_id, version=1):
"""
Removes the database entries and data files related to a record.
:param record_id:
:param version:
:return: True if Successful, False if the record does not exist.
"""
hepdata_submissions = HEPSubmission.query.filter_by(
from yaml import CSafeLoader as yaml_SafeLoader
except ImportError:
from yaml import SafeLoader as yaml_SafeLoader
def _bool_constructor(self, node):
return self.construct_scalar(node)
def _unicode_constructor(self, node):
string_like = self.construct_scalar(node)
return str(string_like)
# Don't follow python bool case
yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor)
# Python2-relevant - become able to resolve "unicode strings"
yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode',
_unicode_constructor)
class DocumentationNotComplete(Exception):
pass
def _save_rename(result, stem, prefix):
result["{0}_{1}".format(prefix, stem)] = stem
def _open_yaml(stream, original_file=None, substitutions_dict={}):
"""
Ensure :mod:`yaml` does not translate on, yes to True, and off, no to False.
.. notes::
In YAML 1.2 the usage of off/on/yes/no was dropped.
"""
bool_values = {
"true": True,
"false": False,
}
v = self.construct_scalar(node)
return bool_values.get(v.lower(), v)
# Disable deserializing yes/no to boolean True/False
_Loader.add_constructor("tag:yaml.org,2002:bool", _construct_bool)
# Disable deserializing date, datetime strings to date, datetime object
_Loader.add_constructor(
"tag:yaml.org,2002:timestamp",
_Loader.yaml_constructors["tag:yaml.org,2002:str"],
)
# Dumper extras
def _represent_path(self, data: Path):
"""
Serialize a `Path` object to a string.
.. warning::
Path("./aaa") serializes to "aaa"
def __init__(self, stream, expand_vars, errors):
SafeLoader.__init__(self, stream)
self.__vars = expand_vars
self.__errors = errors
self.__used_vars = set()
return TemplateString(self.construct_scalar(node))
def _construct_yaml_tuple(self, node):
# Used to convert sequences from lists to tuples. Only applies to lists
# without any nested structures.
seq = self.construct_sequence(node)
if any(isinstance(e, (list, tuple, dict)) for e in seq):
return seq
return tuple(seq)
def _represent_pipeline_config(self, data):
return self.represent_dict(dict(data))
def dump(data, stream=None, Dumper=SafeDumper, **kwargs):
"""See :py:func:`yaml.dump`."""
return yaml.dump(data, stream=stream, Dumper=Dumper, **kwargs)
def load(stream, Loader=SafeLoader):
"""See :py:func:`yaml.load`."""
return yaml.load(stream, Loader)
SafeLoader.add_constructor('tag:yaml.org,2002:seq', _construct_yaml_tuple)
SafeDumper.add_representer(phyre_engine.pipeline.PipelineConfig,
_represent_pipeline_config)
def __init__(self, stream):
SafeLoader.__init__(self, stream)
def construct_odict(load, node):
omap = OrderedDict()
yield omap
if not isinstance(node, yaml.MappingNode):
raise yaml.constructor.ConstructorError(
"while constructing an ordered map", node.start_mark,
"expected a map, but found %s" % node.id, node.start_mark)
for key, value in node.value:
key = load.construct_object(key)
value = load.construct_object(value)
omap[key] = value
Loader.add_constructor(u'tag:yaml.org,2002:map', construct_odict)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
Dumper.add_representer(OrderedDict, ordered_dict_serializer)
# Likewise, when we store unicode objects make sure we don't write
# them with weird YAML tags indicating the Python data type. str-typed
# strings come out fine, but unicode strings come out with unnecessary
# type tags. The easy solution is this:
#
# Dumper.add_representer(unicode, lambda dumper, value:
from __future__ import absolute_import
from __future__ import unicode_literals
from markdown import Extension
from markdown.preprocessors import Preprocessor
import yaml
try:
from yaml import CSafeLoader as Loader
except ImportError:
from yaml import Loader
# Override the default string handling function to always return unicode objects
def construct_yaml_str(self, node):
return self.construct_scalar(node)
Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
class MetaYamlExtension (Extension):
"""Extension for parsing YAML-Metadata with Python-Markdown."""
def extendMarkdown(self, md, md_globals):
"""Add MetaYamlPreprocessor to Markdown instance."""
md.preprocessors.add("meta_yaml", MetaYamlPreprocessor(md), ">meta")
class MetaYamlPreprocessor(Preprocessor):
"""
Get Meta-Data.
A YAML block is delimited by
from yaml import SafeLoader as Loader #pragma: no cover
from hepdata_validator import LATEST_SCHEMA_VERSION
from hepdata_validator.submission_file_validator import SubmissionFileValidator
from hepdata_validator.data_file_validator import DataFileValidator
from hepdata_converter.parsers import Parser, ParsedData, Table
import os, re
# Allow for a bug in PyYAML where numbers like 1e+04 are parsed as strings not as floats.
# See https://stackoverflow.com/a/30462009
# Try replacing PyYAML by ruamel.yaml in future?
Loader.add_implicit_resolver(
u'tag:yaml.org,2002:float',
re.compile(
u'''^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|\\.[0-9_]+(?:[eE][-+][0-9]+)?
|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
|[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$''', re.X), list(u'-+0123456789.'))
class YAML(Parser):
help = 'Parses New HEPData YAML format. Input parameter should be path to ' \
'the directory where submission.yaml file ' \
'is present (or direct filepath to the submission.yaml file)'
def __init__(self, *args, **kwargs):
super(YAML, self).__init__(*args, **kwargs)
self.validator_schema_version = kwargs.get('validator_schema_version',
LATEST_SCHEMA_VERSION)
from horizons.constants import TIER, RES, UNITS, BUILDINGS, PATHS
from horizons.util.yamlcachestorage import YamlCacheStorage
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
# make SafeLoader allow unicode
def construct_yaml_str(self, node):
return self.construct_scalar(node)
SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode',
construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def parse_token(token, token_klass):
"""Helper function that tries to parse a constant name.
Does not do error detection, but passes unparseable stuff through.
Allowed values: integer or token_klass.LIKE_IN_CONSTANTS
@param token_klass: "TIER", "RES", "UNITS" or "BUILDINGS"
"""
classes = {
'TIER': TIER,
'RES': RES,
'UNITS': UNITS,
'BUILDINGS': BUILDINGS
}
def construct_odict(load, node):
omap = OrderedDict()
yield omap
if not isinstance(node, yaml.MappingNode):
raise yaml.constructor.ConstructorError(
"while constructing an ordered map",
node.start_mark,
"expected a map, but found %s" % node.id, node.start_mark
)
for key, value in node.value:
key = load.construct_object(key)
value = load.construct_object(value)
omap[key] = value
Loader.add_constructor(u'tag:yaml.org,2002:map', construct_odict)
def ordered_dict_serializer(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
Dumper.add_representer(OrderedDict, ordered_dict_serializer)
# Likewise, when we store unicode objects make sure we don't write
# them with weird YAML tags indicating the Python data type. The
# standard string type is fine. We should do this:
# Dumper.add_representer(unicode, lambda dumper, value: dumper.represent_scalar(u'tag:yaml.org,2002:str', value))
#
# However, the standard PyYAML representer for strings does something
# weird: if a value cannot be parsed as an integer quotes are omitted.
#
# This is incredibly odd when the value is an integer with a leading
# zero. These values are typically parsed as octal integers, meaning
# quotes would normally be required (that's good). But when the value
except ImportError:
from json import loads
else:
blogfile_exts.insert(0, "yaml")
from functools import partial
try:
# Use libyaml if present, much faster
from yaml import CSafeLoader as _Loader
except ImportError:
from yaml import SafeLoader as _Loader
# Make PyYAML load unicode strings
_Loader.add_constructor(
'tag:yaml.org,2002:str',
lambda self, node: self.construct_scalar(node)
)
loads = partial(_loads, Loader=_Loader)
inifile = PIniFile("simpleblog", [
('source', [
('encoding', INI_STRING, 'utf-8')
])
])
def blogdata(data, encoding=inifile.source_encoding):
return decode(data, encoding)
def load(self, headers, file):
loader = YamlLoader(file)
return ([object.get(header, Undefined) for header in headers]
for object in loader.get_data())
def __init__(self, stream, expand_vars, errors):
SafeLoader.__init__(self, stream)
self.__vars = expand_vars
self.__errors = errors
self.__used_vars = set()
def construct_tuple(loader, node):
return tuple(Loader.construct_sequence(loader, node))
def entity_serializer(dumper, entity):
return dumper.represent_data(entity.to_dict())
log.debug('registering smisk.mvc.model.Entity YAML serializer (W)')
Dumper.add_multi_representer(Entity, entity_serializer)
# support for serializing data:
def data_serializer(dumper, dat):
return dumper.represent_scalar(u'!data', dat.encode())
def data_unserializer(loader, datatype, node):
return opaque_data.decode(node.value)
log.debug('registering smisk.serialization.data YAML serializer (RW)')
Dumper.add_multi_representer(opaque_data, data_serializer)
Loader.add_multi_constructor(u'!data', data_unserializer)
if __name__ == '__main__':
data = {
'message': 'Hello worlds',
'internets': [
'interesting',
'lolz',
42.0,
{
'abcdata': opaque_data('xyz detta överförs binärt'),
'tubes': [1,3,16,18,24],
'persons': True,
u'me agåain': {
'message': 'Hello worlds',
from .constants import (PKG_MANAGER_TO_SYSTEM, JINJA_MACROS_BASE_DEFINITIONS,
JINJA_MACROS_HIGHLEVEL_DEFINITIONS)
from .constants import DEFAULT_UID_MIN
try:
from yaml import CSafeLoader as yaml_SafeLoader
except ImportError:
from yaml import SafeLoader as yaml_SafeLoader
def _bool_constructor(self, node):
return self.construct_scalar(node)
# Don't follow python bool case
yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor)
def _save_rename(result, stem, prefix):
result["{0}_{1}".format(prefix, stem)] = stem
def _open_yaml(stream, original_file=None):
"""
Open given file-like object and parse it as YAML.
Optionally, pass the path to the original_file for better error handling
when the file contents are passed.
Return None if it contains "documentation_complete" key set to "false".
"""
JINJA_MACROS_BASE_DEFINITIONS,
JINJA_MACROS_HIGHLEVEL_DEFINITIONS)
from .constants import DEFAULT_UID_MIN
try:
from yaml import CSafeLoader as yaml_SafeLoader
except ImportError:
from yaml import SafeLoader as yaml_SafeLoader
def _bool_constructor(self, node):
return self.construct_scalar(node)
# Don't follow python bool case
yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor)
def _save_rename(result, stem, prefix):
result["{0}_{1}".format(prefix, stem)] = stem
def _open_yaml(stream, original_file=None, substitutions_dict={}):
"""
Open given file-like object and parse it as YAML.
Optionally, pass the path to the original_file for better error handling
when the file contents are passed.
Return None if it contains "documentation_complete" key set to "false".
"""
import yaml
import threading
import traceback
import logging
from horizons.constants import TIER, RES, UNITS, BUILDINGS, PATHS
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
# make SafeLoader allow unicode
def construct_yaml_str(self, node):
return self.construct_scalar(node)
SafeLoader.add_constructor(u'tag:yaml.org,2002:python/unicode', construct_yaml_str)
SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
def parse_token(token, token_klass):
"""Helper function that tries to parse a constant name.
Does not do error detection, but passes unparseable stuff through.
Allowed values: integer or token_klass.LIKE_IN_CONSTANTS
@param token_klass: "TIER", "RES", "UNITS" or "BUILDINGS"
"""
classes = {'TIER': TIER, 'RES': RES, 'UNITS': UNITS, 'BUILDINGS': BUILDINGS}
if isinstance(token, unicode):
if token.startswith(token_klass):
try:
return getattr( classes[token_klass], token.split(".", 2)[1])
from yaml import load as _loads
except ImportError:
from json import loads
else:
blogfile_exts.insert(0, "yaml")
from functools import partial
try:
# Use libyaml if present, much faster
from yaml import CSafeLoader as _Loader
except ImportError:
from yaml import SafeLoader as _Loader
# Make PyYAML load unicode strings
_Loader.add_constructor('tag:yaml.org,2002:str',
lambda self, node: self.construct_scalar(node))
loads = partial(_loads, Loader=_Loader)
inifile = PIniFile("simpleblog",
[('source', [('encoding', INI_STRING, 'utf-8')])])
def blogdata(data, encoding=inifile.source_encoding):
return decode(data, encoding)
def read_blogfile(filename):
with open(filename, 'rb') as f:
data = f.read()
return blogdata(data)