def read_config(self, fields: List[OptionInstance]) -> None: pure = True # get a scanner to read the yaml with open(self.__filename, "r+t") as file: options = load(file, Loader=Loader) if len(fields) > 0: for field in fields: if options is not None and field.name in options: # get the type field.return_value(options[field.name]) else: field.return_default() pure = False # check if there were any missing members if not pure: # write back to yaml with open(self.__filename, "w+t") as file: dumper = Dumper(stream=file) try: dumper.open() mapping = {field.name: field.value for field in fields} dumper.serialize(dumper.represent_mapping(None, mapping)) dumper.close() finally: dumper.dispose() raise OptionsError( 'Config is missing some members! Fill them out and try again.')
def __init__(self, stream, include_virtual_fields=True, include_type_field=True, *args, **kwargs): SafeDumper.__init__(self, stream, *args, **kwargs) self.include_virtual_fields = include_virtual_fields self.include_type_field = include_type_field
def dump(self, headers, data, file): dumper = YamlDumper(file, **self.serializer_parameters) try: dumper.open() dumper.represent(({k: v for k, v in zip(headers, row)} for row in data)) dumper.close() finally: dumper.dispose()
def ordered_dumper(dumper, data): """ Ensure that maps are always dumped in the items order. """ return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items()) SaneDumper.add_representer(OrderedDict, ordered_dumper) def null_dumper(dumper, value): """ Always dump nulls as empty string. """ return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'') SafeDumper.add_representer(type(None), null_dumper) def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'): """ Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in the sanest and most readable way. """ if not isinstance(value, basestring): value = repr(value) if isinstance(value, str): value = value.decode('utf-8') style = None multilines = '\n' in value
def split_files(file_location, output_location, archive_location=None): """ :param file_location: input yaml file location :param output_location: output directory path :param archive_location: if present will create a zipped representation of the split files """ last_updated = datetime.now() try: file_documents = yaml.load_all(open(file_location, 'r'), Loader=Loader) # make a submission directory where all the files will be stored. # delete a directory in the event that it exists. if os.path.exists(output_location): shutil.rmtree(output_location) os.makedirs(output_location) with open(os.path.join(output_location, "submission.yaml"), 'w') as submission_yaml: for document in file_documents: if not document: continue elif "name" not in document: if "dateupdated" in document: try: last_updated = parse(document['dateupdated'], dayfirst=True) except ValueError as ve: last_updated = datetime.now() write_submission_yaml_block(document, submission_yaml) else: file_name = document["name"].replace(' ', '') + ".yaml" document["data_file"] = file_name with open(os.path.join(output_location, file_name), 'w') as data_file: Dumper.add_representer(str, str_presenter) yaml.dump( { "independent_variables": cleanup_data_yaml( document["independent_variables"]), "dependent_variables": cleanup_data_yaml( document["dependent_variables"]) }, data_file, allow_unicode=True, Dumper=Dumper) write_submission_yaml_block(document, submission_yaml, type="record") if archive_location: if os.path.exists(archive_location): os.remove(archive_location) zipf = zipfile.ZipFile(archive_location, 'w') os.chdir(output_location) zipdir(".", zipf) zipf.close() except yaml.scanner.ScannerError as se: return se, last_updated except yaml.parser.ParserError as pe: return pe, last_updated except Exception as e: log.error('Error parsing %s, %s', file_location, e.message) return e, last_updated return None, last_updated
def write_submission_yaml_block(document, submission_yaml, type="info"): submission_yaml.write("---\n") cleanup_yaml(document, type) Dumper.add_representer(str, str_presenter) yaml.dump(document, submission_yaml, allow_unicode=True, Dumper=Dumper) submission_yaml.write("\n")
return self.represent_scalar("tag:yaml.org,2002:str", str(data)) def _represent_ordered_dict(self, data): """ Serialize OrderedDict to a yaml map :param data: [description] :type data: collections.OrderedDict """ return self.represent_mapping("tag:yaml.org,2002:map", data.items()) # Serializing Python `Path` objects to `str` # NOTE: Path("./aaa") serializes to "aaa" _Dumper.add_multi_representer(Path, _represent_path) _Dumper.add_representer(OrderedDict, _represent_ordered_dict) load = partial(_yaml.load, Loader=_Loader) def loads(s: str, *args, **kwargs) -> Dict: """ Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance containing a YAML document) to a Python dictionary. [extended_summary] :param s: [description] :type s: str :return: [description]
def __init__(self, *args, **kwargs): super(YAML, self).__init__(single_file_output=True, *args, **kwargs) Dumper.add_representer(str, str_presenter)
from hepdata_lib.root_utils import RootFileReader MAPPING_TAG = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG def dict_representer(dumper, data): """represent dict.""" return dumper.represent_dict(data.items()) def dict_constructor(loader, node): """construct dict.""" return defaultdict(loader.construct_pairs(node)) yaml.add_representer(defaultdict, SafeRepresenter.represent_dict) Dumper.add_representer(defaultdict, dict_representer) Loader.add_constructor(MAPPING_TAG, dict_constructor) Dumper.add_representer(str, SafeRepresenter.represent_str) # Display deprecation warnings warnings.filterwarnings("always", category=DeprecationWarning, module="hepdata_lib") __version__ = "0.4.1" class Variable(object): """A Variable is a wrapper for a list of values + some meta data.""" # pylint: disable=too-many-instance-attributes # Eight is reasonable in this case.
Ensure that maps are always dumped in the items order. """ return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.items()) SaneDumper.add_representer(OrderedDict, ordered_dumper) def null_dumper(dumper, value): # @UnusedVariable """ Always dump nulls as empty string. """ return dumper.represent_scalar(u'tag:yaml.org,2002:null', u'') SafeDumper.add_representer(type(None), null_dumper) def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'): """ Ensure that all scalars are dumped as UTF-8 unicode, folded and quoted in the sanest and most readable way. """ style = None if not isinstance(value, basestring): value = repr(value) if isinstance(value, str): value = value.decode('utf-8')
try: from yaml import CSafeDumper as SafeDumper except ImportError: from yaml import SafeDumper class SafeDumper(SafeDumper): def represent_decimal(self, data): return self.represent_scalar('tag:yaml.org,2002:str', str(data)) def represent_ordered_dict(self, data): return self.represent_mapping('tag:yaml.org,2002:map', data.items()) SafeDumper.add_representer(Decimal, SafeDumper.represent_decimal) SafeDumper.add_representer(OrderedDict, SafeDumper.represent_ordered_dict) class PrettyYamlSerializer(Serializer): def end_serialization(self): objects = [ OrderedDict([ ( 'model', i['model'], ), ( 'pk', i['pk'], ),
def add_representer(klass: Any, representer: Any) -> None: """Add to representer to the dumper.""" FastestAvailableSafeDumper.add_representer(klass, representer)
return (st, None) else: return ((st,), None) # Register if we have a backing YAML implementation if yaml is not None: serializers.register(YAMLSerializer) # support for serializing Entities: from smisk.mvc.model import Entity def entity_serializer(dumper, entity): return dumper.represent_data(entity.to_dict()) log.debug('registering smisk.mvc.model.Entity YAML serializer (W)') Dumper.add_multi_representer(Entity, entity_serializer) # support for serializing data: def data_serializer(dumper, dat): return dumper.represent_scalar(u'!data', dat.encode()) def data_unserializer(loader, datatype, node): return opaque_data.decode(node.value) log.debug('registering smisk.serialization.data YAML serializer (RW)') Dumper.add_multi_representer(opaque_data, data_serializer) Loader.add_multi_constructor(u'!data', data_unserializer) if __name__ == '__main__': data = {
# -*- coding:utf-8 -*- from __future__ import absolute_import, unicode_literals from types import GeneratorType try: from yaml import CSafeLoader as YamlLoader, CSafeDumper as YamlDumper except ImportError: from yaml import SafeLoader as YamlLoader, SafeDumper as YamlDumper from django.utils.six.moves import zip from yepes.contrib.datamigrations.serializers import Serializer from yepes.types import Undefined YamlDumper.add_representer(GeneratorType, YamlDumper.represent_list) class YamlSerializer(Serializer): def __init__(self, **serializer_parameters): defaults = { 'allow_unicode': True, 'encoding': None, 'explicit_end': False, 'explicit_start': False, 'width': 1000, } defaults.update(serializer_parameters) super(YamlSerializer, self).__init__(**defaults) def dump(self, headers, data, file):
return TemplateString(self.construct_scalar(node)) def _construct_yaml_tuple(self, node): # Used to convert sequences from lists to tuples. Only applies to lists # without any nested structures. seq = self.construct_sequence(node) if any(isinstance(e, (list, tuple, dict)) for e in seq): return seq return tuple(seq) def _represent_pipeline_config(self, data): return self.represent_dict(dict(data)) def dump(data, stream=None, Dumper=SafeDumper, **kwargs): """See :py:func:`yaml.dump`.""" return yaml.dump(data, stream=stream, Dumper=Dumper, **kwargs) def load(stream, Loader=SafeLoader): """See :py:func:`yaml.load`.""" return yaml.load(stream, Loader) SafeLoader.add_constructor('tag:yaml.org,2002:seq', _construct_yaml_tuple) SafeDumper.add_representer(phyre_engine.pipeline.PipelineConfig, _represent_pipeline_config)