def represent_string(
dumper: yaml.Dumper,
data: str,
tag: str = BaseResolver.DEFAULT_SCALAR_TAG,
basicsep: str = "",
) -> yaml.ScalarNode:
"""Override the normal string emission rules to produce the most readable text output.
Args:
data: The string to dump
tag: (Optional) The YAML tag to use for this scalar. Defaults to the
default YAML scalar tag, which does not get printed.
basicsep: (Optional) The scalar string separator to use for "simple"
strings (ie. strings where there isn't a more specific rule)
"""
# TODO test cases
if "\n" in data:
# '|' style means literal block style, so line breaks and formatting are retained.
# This will be especially handy for inline code.
return dumper.represent_scalar(tag, data, "|")
elif len(data) > 65:
# Longer lines will be automatically folded (ie. have line breaks
# inserted) by PyYAML, which is likely to cause confusion. We
# compromise by using the literal block style ('|'), which doesn't
# fold, but does require some special block indicators.
# TODO need some way to allow folding. Having a helper function probably isn't really good enough. perhaps have the reflow function return a special subclass of string which we can detect here
return dumper.represent_scalar(tag, data, "|")
else:
return dumper.represent_scalar(tag, data, basicsep)
def create_folders(options, structures, publication_template, root=''):
out_format = 'json'
Dumper.add_representer(collections.OrderedDict, dict_representer)
for key in structures:
if isinstance(structures[key], dict):
d = Path(root).joinpath(key)
Path(d).mkdir(parents=True, exist_ok=True)
if Path(root).parent.as_posix() == '.':
# Have to explicitly convert Path to str
# to work under python 3.4
with open(str(Path(root).joinpath('publication.txt')),
'w') as outfile:
yaml.dump(publication_template,
outfile,
indent=4,
Dumper=Dumper)
if options.energy_corrections:
with open(
str(Path(root).joinpath('energy_corrections.txt')),
'w') as outfile:
yaml.dump(options.energy_corrections, outfile)
create_folders(options,
structures[key],
publication_template={},
root=d)
else:
ase.io.write(
str(Path(root).joinpath(key + '.' + out_format)),
structures[key],
format=out_format,
)
def redump(stream):
parser = Parser(StringStream(stream))
parser.dictionary = OrderedDict
docs = list(iter(parser))
dumper = Dumper()
dumper.alphaSort = 0
return dumper.dump(*docs)
def __init__(cls, name, bases, kwds):
"""This overlaps quite a bit with YAMLObjectMetaclass."""
if name != "ManagementObject":
yaml_tag = u"tag:yaml.org,2002:es.bsc.%s" % (cls.__module__)
cls.yaml_loader = Loader
cls.yaml_tag = yaml_tag # used by `ManagementObject.to_yaml`
logger.trace("YAML TAG : %s", yaml_tag)
Loader.add_constructor(yaml_tag, cls.from_yaml)
Dumper.add_representer(cls, cls.to_yaml)
super(ManagementMetaClass, cls).__init__(name, bases, kwds)
def dump_yaml(data):
import io
from yaml import Dumper
stream = io.StringIO()
dumper = Dumper(stream)
try:
dumper.open()
dumper.represent(data)
dumper.close()
finally:
dumper.dispose()
return stream.getvalue()
def repr_str(dumper: Dumper, data: str) -> str:
"""A YAML Representer that handles strings, breaking multi-line strings into something a lot
more readable in the yaml output. This is useful for representing long, multiline strings in
templates or in stack parameters.
:param dumper: The Dumper that is being used to serialize this object
:param data: The string to serialize
:return: The represented string
"""
if '\n' in data:
return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
return dumper.represent_str(data)
def save_yaml_opts(path_yaml, opts):
# Warning: copy is not nested
options = copy.copy(opts)
# https://gist.github.com/oglops/c70fb69eef42d40bed06
def dict_representer(dumper, data):
return dumper.represent_dict(data.items())
Dumper.add_representer(Dict, dict_representer)
with open(path_yaml, 'w') as yaml_file:
yaml.dump(options, yaml_file, Dumper=Dumper, default_flow_style=False)
def _uuid_representer(dumper: yaml.Dumper, data: uuid.UUID) -> yaml.Node:
"""Generate YAML representation for UUID.
This produces a scalar node with a tag "!uuid" and value being a regular
string representation of UUID.
"""
return dumper.represent_scalar("!uuid", str(data))
def as_yaml_node(self, dumper: yaml.Dumper) -> yaml.Node:
# Ideally, we would create a tag that contains the object name
# (eg. "!Bucket") for completeness. Unfortunately, there is no way
# to then prevent the YAML dumper from printing tags unless it
# determines that the tag is implicit (ie. the default tag for a
# mapping node is the tag being used), so we end up just using the
# default tag.
# TODO investigate hacking a Dumper subclass like we did for aliasing
tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
# Get all cloud formation attributes that are set, in sorted order
attributes = [(key, self[key]) for key in self
if self.is_attribute_set(key)]
# Create neater YAML by filtering out empty blocks at this level
attributes = [(key, value) for key, value in attributes
if is_non_empty_attribute(value)]
# Create neater YAML by filtering out empty entries in sub-lists
attributes = [(key, remove_empty_values_from_attribute(value))
for key, value in attributes]
# Represent this object as a mapping of it's AWS attributes.
# Note that `represent_mapping` works on a list of 2-tuples, not a map!
return dumper.represent_mapping(tag, attributes)
def str_representer(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode:
with_quotes = yaml_is_bool(data) or is_int(data) or is_float(data)
return dumper.represent_scalar(
yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG,
data,
style=("'" if with_quotes else None),
)
def read_config(self, fields: List[OptionInstance]) -> None:
pure = True
# get a scanner to read the yaml
with open(self.__filename, "r+t") as file:
options = load(file, Loader=Loader)
if len(fields) > 0:
for field in fields:
if options is not None and field.name in options:
# get the type
field.return_value(options[field.name])
else:
field.return_default()
pure = False
# check if there were any missing members
if not pure:
# write back to yaml
with open(self.__filename, "w+t") as file:
dumper = Dumper(stream=file)
try:
dumper.open()
mapping = {field.name: field.value for field in fields}
dumper.serialize(dumper.represent_mapping(None, mapping))
dumper.close()
finally:
dumper.dispose()
raise OptionsError(
'Config is missing some members! Fill them out and try again.')
def _InferenceAPI_dumper(dumper: yaml.Dumper, api: InferenceAPI) -> yaml.Node:
return dumper.represent_dict({
"route": api.route,
"doc": api.doc,
"input": api.input.__class__.__name__,
"output": api.output.__class__.__name__,
})
def print_yaml(disasm, stream=sys.stdout):
from yaml import Dumper
dumper = Dumper(stream)
# Force a Git-friendly output format:
# https://stackoverflow.com/a/8641732
def strdump(dumper, data):
return dumper.represent_scalar('tag:yaml.org,2002:str',
data,
style='|')
dumper.add_representer(str, strdump)
dumper.open()
dumper.represent(disasm)
dumper.close()
def arrow_representer(dumper: yaml.Dumper, data: arrow.Arrow) -> str:
"""
Represent an `arrow.arrow.Arrow` object as a scalar in ISO format.
! '2013-05-07T04:24:24+00:00'
"""
return dumper.represent_scalar("!", data.isoformat("T"))
def _get_string_node(
self, dumper: yaml.Dumper, value: Union["_Function", str], tag: str
) -> yaml.Node:
"""Get a PyYAML node for a function that returns a string.
Lots of the intrinsic functions take a single string as the argument,
which we echo directly into the output. However, in CloudFormation,
this string could be a reference to another function rather than a
literal string, and that needs to be handled specially.
"""
if isinstance(value, _Function):
return dumper.represent_dict({f"Fn::{tag}": value})
else:
# TODO be able to control the scalar output, perhaps
# TODO use represent_string instead, perhaps
return dumper.represent_scalar(f"!{tag}", value, style="")
def yaml_representer(dumper: yaml.Dumper, instance):
output = []
if instance.comments is not None and len(instance.comments) > 0:
output.append(('comments', instance.comments))
if instance.additional_urls is not None and len(
instance.additional_urls) > 0:
output.append(('additional_urls', instance.additional_urls))
return dumper.represent_dict(output)
def ordered_dict_representer(dumper: yaml.Dumper,
data: Mapping) -> yaml.MappingNode:
'''
Register OrderedDict in order to be able to dump it.
'''
return dumper.represent_mapping(u'tag:yaml.org,2002:map',
data.items(),
flow_style=False) # block flow style
def _represent_root_certificate_entry(dumper: yaml.Dumper, entry: RootCertificateRecord) -> yaml.Node:
# TODO(AD): this seems to maintain order for fields because dicts in Python 3.6 keep the order - it "should not be
# relied upon" but let's rely on it anyway for now
final_dict = {
'subject_name': entry.subject_name,
'fingerprint': entry.hex_fingerprint,
}
return dumper.represent_dict(final_dict.items())
def enum_to_string_representer(dumper: yaml.Dumper,
data: enum.Enum) -> yaml.ScalarNode:
'''
Register some sort of enum as being serialized by `str()`.
'''
value = str(data)
return dumper.represent_scalar(u'tag:yaml.org,2002:str',
value)
def literal_string_representer(dumper: yaml.Dumper,
data: Union[LiteralString, str]
) -> yaml.ScalarNode:
'''
Register FoldedString as the correct style of literal.
'''
return dumper.represent_scalar(u'tag:yaml.org,2002:str',
str(data),
style='|')
def string_representer(
dumper: yaml.Dumper,
data: str,
) -> yaml.Dumper.represent_scalar:
"""
Add a custom string representer to use block literals for multiline strings.
Args:
dumper: A YAML dumper.
data: A data string.
Returns:
YAML dumper.represent_scalar.
"""
if len(data.splitlines()) > 1:
return dumper.represent_scalar(tag='tag:yaml.org,2002:str',
value=data,
style='|')
return dumper.represent_scalar(tag='tag:yaml.org,2002:str', value=data)
def root_representer(dumper: yaml.Dumper, data: YAMLRoot):
""" YAML callback -- used to filter out empty values (None, {}, [] and false)
@param dumper: data dumper
@param data: data to be dumped
@return:
"""
rval = dict()
for k, v in data.__dict__.items():
if not k.startswith('_') and v is not None and (not isinstance(v, (dict, list)) or v):
rval[k] = v
return dumper.represent_data(rval)
def enum_string_value_representer(dumper: yaml.Dumper,
data: enum.Enum) -> yaml.ScalarNode:
'''
Register some sort of enum as being serialized by str value.
'''
value = data.value
if not isinstance(value, str):
# Hm... can't log... don't want to print...
# Just represent this as best we can?
value = str(value)
return dumper.represent_scalar(u'tag:yaml.org,2002:str',
value)
def _range_representer(dumper: yaml.Dumper, data: Range) -> yaml.Node:
begin, end = data
# pyyaml doesn't output timestamps in flow style as timestamps(?)
if isinstance(begin, datetime):
begin = begin.isoformat()
if isinstance(end, datetime):
end = end.isoformat()
return dumper.represent_mapping(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
(('begin', begin), ('end', end)),
flow_style=True)
def repr_odict(dumper: Dumper, data: ODict) -> str:
"""A YAML Representer for cfn-flip's ODict objects.
ODicts are a variation on OrderedDicts for that library. Since the Diff command makes extensive
use of ODicts, they can end up in diff output and the PyYaml library doesn't otherwise calls the
dicts like !!ODict, which looks weird. We can just treat them like normal dicts when we serialize
them, though.
:param dumper: The Dumper that is being used to serialize this object
:param data: The ODict object to serialize
:return: The serialized ODict
"""
return dumper.represent_dict(data)
def to_yaml(cls, dumper: yaml.Dumper, data: astropy.time.Time) -> Any:
"""Convert astropy Time object into YAML format.
Parameters
----------
dumper : `yaml.Dumper`
YAML dumper instance.
data : `astropy.time.Time`
Data to be converted.
"""
if data is not None:
# we store time in ISO format but we need full nanosecond
# precision so we have to construct intermediate instance to make
# sure its precision is set correctly.
data = astropy.time.Time(data.tai, precision=9)
data = data.to_value("iso")
return dumper.represent_scalar(cls.yaml_tag, data)
def root_representer(dumper: yaml.Dumper, data: YAMLRoot):
""" YAML callback -- used to filter out empty values (None, {}, [] and false)
@param dumper: data dumper
@param data: data to be dumped
@return:
"""
# TODO: Figure out how to import EnumDefinition here
# elif isinstance(v, EnumDefinition):
from biolinkml.utils.enumerations import EnumDefinitionImpl
if isinstance(data, EnumDefinitionImpl):
data = data.code
rval = dict()
for k, v in data.__dict__.items():
if not k.startswith('_') and v is not None and (
not isinstance(v, (dict, list)) or v):
rval[k] = v
return dumper.represent_data(rval)
def _represent_trust_store(dumper: yaml.Dumper, store: TrustStore) -> yaml.Node:
# Always sort the certificates alphabetically so it is easy to diff the list
sorted_trusted_certs = sorted(store.trusted_certificates, key=attrgetter('subject_name', 'hex_fingerprint'))
sorted_blocked_certs = sorted(store.blocked_certificates, key=attrgetter('subject_name', 'hex_fingerprint'))
# TODO(AD): this seems to maintain order for fields because dicts in Python 3.6 keep the order - it "should not be
# relied upon" but let's rely on it anyway for now
final_dict = {
'platform': store.platform.name,
'version': store.version,
'url': store.url,
'date_fetched': store.date_fetched,
'trusted_certificates_count': store.trusted_certificates_count,
'trusted_certificates': sorted_trusted_certs,
'blocked_certificates_count': store.blocked_certificates_count,
'blocked_certificates': sorted_blocked_certs,
}
return dumper.represent_dict(final_dict.items())
def _yaml_represent_OrderedDict(dumper: yaml.Dumper, data: OrderedDict):
return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map',
[(dumper.represent_data(k), dumper.represent_data(v))
for k, v in data.items()])
def redump(stream):
docs = list(loadOrdered(stream))
dumper = Dumper()
dumper.alphaSort = 0
return dumper.dump(*docs)
def pkg_representer(dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:
"""Represent Packages as a simple dict"""
return dumper.represent_mapping("lsst.utils.packages.Packages",
data,
flow_style=None)
def _ModelInfo_dumper(dumper: yaml.Dumper, info: ModelInfo) -> yaml.Node:
return dumper.represent_dict(info.to_dict())
def bytes_representer(dumper: yaml.Dumper, obj):
try:
return dumper.represent_str(obj.decode('utf-8'))
except UnicodeError:
return dumper.represent_str(binascii.hexlify(obj).decode('utf-8'))
