def read_config(self, fields: List[OptionInstance]) -> None:
pure = True
# get a scanner to read the yaml
with open(self.__filename, "r+t") as file:
options = load(file, Loader=Loader)
if len(fields) > 0:
for field in fields:
if options is not None and field.name in options:
# get the type
field.return_value(options[field.name])
else:
field.return_default()
pure = False
# check if there were any missing members
if not pure:
# write back to yaml
with open(self.__filename, "w+t") as file:
dumper = Dumper(stream=file)
try:
dumper.open()
mapping = {field.name: field.value for field in fields}
dumper.serialize(dumper.represent_mapping(None, mapping))
dumper.close()
finally:
dumper.dispose()
raise OptionsError(
'Config is missing some members! Fill them out and try again.')
def __init__(self,
stream,
include_virtual_fields=True,
include_type_field=True,
*args,
**kwargs):
SafeDumper.__init__(self, stream, *args, **kwargs)
self.include_virtual_fields = include_virtual_fields
self.include_type_field = include_type_field
def dump(self, headers, data, file):
dumper = YamlDumper(file, **self.serializer_parameters)
try:
dumper.open()
dumper.represent(({k: v
for k, v in zip(headers, row)} for row in data))
dumper.close()
finally:
dumper.dispose()
def ordered_dumper(dumper, data):
"""
Ensure that maps are always dumped in the items order.
"""
return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items())
SaneDumper.add_representer(OrderedDict, ordered_dumper)
def null_dumper(dumper, value):
"""
Always dump nulls as empty string.
"""
return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'')
SafeDumper.add_representer(type(None), null_dumper)
def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'):
"""
Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in
the sanest and most readable way.
"""
if not isinstance(value, basestring):
value = repr(value)
if isinstance(value, str):
value = value.decode('utf-8')
style = None
multilines = '\n' in value
def __init__(self, stream, include_virtual_fields=True, include_type_field=True, *args, **kwargs):
SafeDumper.__init__(self, stream, *args, **kwargs)
self.include_virtual_fields = include_virtual_fields
self.include_type_field = include_type_field
def split_files(file_location, output_location, archive_location=None):
"""
:param file_location: input yaml file location
:param output_location: output directory path
:param archive_location: if present will create a zipped
representation of the split files
"""
last_updated = datetime.now()
try:
file_documents = yaml.load_all(open(file_location, 'r'), Loader=Loader)
# make a submission directory where all the files will be stored.
# delete a directory in the event that it exists.
if os.path.exists(output_location):
shutil.rmtree(output_location)
os.makedirs(output_location)
with open(os.path.join(output_location, "submission.yaml"),
'w') as submission_yaml:
for document in file_documents:
if not document:
continue
elif "name" not in document:
if "dateupdated" in document:
try:
last_updated = parse(document['dateupdated'],
dayfirst=True)
except ValueError as ve:
last_updated = datetime.now()
write_submission_yaml_block(document, submission_yaml)
else:
file_name = document["name"].replace(' ', '') + ".yaml"
document["data_file"] = file_name
with open(os.path.join(output_location, file_name),
'w') as data_file:
Dumper.add_representer(str, str_presenter)
yaml.dump(
{
"independent_variables":
cleanup_data_yaml(
document["independent_variables"]),
"dependent_variables":
cleanup_data_yaml(
document["dependent_variables"])
},
data_file,
allow_unicode=True,
Dumper=Dumper)
write_submission_yaml_block(document,
submission_yaml,
type="record")
if archive_location:
if os.path.exists(archive_location):
os.remove(archive_location)
zipf = zipfile.ZipFile(archive_location, 'w')
os.chdir(output_location)
zipdir(".", zipf)
zipf.close()
except yaml.scanner.ScannerError as se:
return se, last_updated
except yaml.parser.ParserError as pe:
return pe, last_updated
except Exception as e:
log.error('Error parsing %s, %s', file_location, e.message)
return e, last_updated
return None, last_updated
def write_submission_yaml_block(document, submission_yaml, type="info"):
submission_yaml.write("---\n")
cleanup_yaml(document, type)
Dumper.add_representer(str, str_presenter)
yaml.dump(document, submission_yaml, allow_unicode=True, Dumper=Dumper)
submission_yaml.write("\n")
return self.represent_scalar("tag:yaml.org,2002:str", str(data))
def _represent_ordered_dict(self, data):
"""
Serialize OrderedDict to a yaml map
:param data: [description]
:type data: collections.OrderedDict
"""
return self.represent_mapping("tag:yaml.org,2002:map", data.items())
# Serializing Python `Path` objects to `str`
# NOTE: Path("./aaa") serializes to "aaa"
_Dumper.add_multi_representer(Path, _represent_path)
_Dumper.add_representer(OrderedDict, _represent_ordered_dict)
load = partial(_yaml.load, Loader=_Loader)
def loads(s: str, *args, **kwargs) -> Dict:
"""
Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a YAML document) to a Python dictionary.
[extended_summary]
:param s: [description]
:type s: str
:return: [description]
def __init__(self, *args, **kwargs):
super(YAML, self).__init__(single_file_output=True, *args, **kwargs)
Dumper.add_representer(str, str_presenter)
from hepdata_lib.root_utils import RootFileReader
MAPPING_TAG = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
"""represent dict."""
return dumper.represent_dict(data.items())
def dict_constructor(loader, node):
"""construct dict."""
return defaultdict(loader.construct_pairs(node))
yaml.add_representer(defaultdict, SafeRepresenter.represent_dict)
Dumper.add_representer(defaultdict, dict_representer)
Loader.add_constructor(MAPPING_TAG, dict_constructor)
Dumper.add_representer(str,
SafeRepresenter.represent_str)
# Display deprecation warnings
warnings.filterwarnings("always", category=DeprecationWarning, module="hepdata_lib")
__version__ = "0.4.1"
class Variable(object):
"""A Variable is a wrapper for a list of values + some meta data."""
# pylint: disable=too-many-instance-attributes
# Eight is reasonable in this case.
Ensure that maps are always dumped in the items order.
"""
return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items())
SaneDumper.add_representer(OrderedDict, ordered_dumper)
def null_dumper(dumper, value): # @UnusedVariable
"""
Always dump nulls as empty string.
"""
return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'')
SafeDumper.add_representer(type(None), null_dumper)
def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'):
"""
Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in
the sanest and most readable way.
"""
style = None
if not isinstance(value, basestring):
value = repr(value)
if isinstance(value, str):
value = value.decode('utf-8')
try:
from yaml import CSafeDumper as SafeDumper
except ImportError:
from yaml import SafeDumper
class SafeDumper(SafeDumper):
def represent_decimal(self, data):
return self.represent_scalar('tag:yaml.org,2002:str', str(data))
def represent_ordered_dict(self, data):
return self.represent_mapping('tag:yaml.org,2002:map', data.items())
SafeDumper.add_representer(Decimal, SafeDumper.represent_decimal)
SafeDumper.add_representer(OrderedDict, SafeDumper.represent_ordered_dict)
class PrettyYamlSerializer(Serializer):
def end_serialization(self):
objects = [
OrderedDict([
(
'model',
i['model'],
),
(
'pk',
i['pk'],
),
def add_representer(klass: Any, representer: Any) -> None:
"""Add to representer to the dumper."""
FastestAvailableSafeDumper.add_representer(klass, representer)
return (st, None)
else:
return ((st,), None)
# Register if we have a backing YAML implementation
if yaml is not None:
serializers.register(YAMLSerializer)
# support for serializing Entities:
from smisk.mvc.model import Entity
def entity_serializer(dumper, entity):
return dumper.represent_data(entity.to_dict())
log.debug('registering smisk.mvc.model.Entity YAML serializer (W)')
Dumper.add_multi_representer(Entity, entity_serializer)
# support for serializing data:
def data_serializer(dumper, dat):
return dumper.represent_scalar(u'!data', dat.encode())
def data_unserializer(loader, datatype, node):
return opaque_data.decode(node.value)
log.debug('registering smisk.serialization.data YAML serializer (RW)')
Dumper.add_multi_representer(opaque_data, data_serializer)
Loader.add_multi_constructor(u'!data', data_unserializer)
if __name__ == '__main__':
data = {
# -*- coding:utf-8 -*-
from __future__ import absolute_import, unicode_literals
from types import GeneratorType
try:
from yaml import CSafeLoader as YamlLoader, CSafeDumper as YamlDumper
except ImportError:
from yaml import SafeLoader as YamlLoader, SafeDumper as YamlDumper
from django.utils.six.moves import zip
from yepes.contrib.datamigrations.serializers import Serializer
from yepes.types import Undefined
YamlDumper.add_representer(GeneratorType, YamlDumper.represent_list)
class YamlSerializer(Serializer):
def __init__(self, **serializer_parameters):
defaults = {
'allow_unicode': True,
'encoding': None,
'explicit_end': False,
'explicit_start': False,
'width': 1000,
}
defaults.update(serializer_parameters)
super(YamlSerializer, self).__init__(**defaults)
def dump(self, headers, data, file):
return TemplateString(self.construct_scalar(node))
def _construct_yaml_tuple(self, node):
# Used to convert sequences from lists to tuples. Only applies to lists
# without any nested structures.
seq = self.construct_sequence(node)
if any(isinstance(e, (list, tuple, dict)) for e in seq):
return seq
return tuple(seq)
def _represent_pipeline_config(self, data):
return self.represent_dict(dict(data))
def dump(data, stream=None, Dumper=SafeDumper, **kwargs):
"""See :py:func:`yaml.dump`."""
return yaml.dump(data, stream=stream, Dumper=Dumper, **kwargs)
def load(stream, Loader=SafeLoader):
"""See :py:func:`yaml.load`."""
return yaml.load(stream, Loader)
SafeLoader.add_constructor('tag:yaml.org,2002:seq', _construct_yaml_tuple)
SafeDumper.add_representer(phyre_engine.pipeline.PipelineConfig,
_represent_pipeline_config)
