def test_file_output(unicode_filename, verbose=False):
yaml = YAML(typ='safe', pure=True)
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
handle, filename = tempfile.mkstemp()
os.close(handle)
try:
stream = StringIO()
yaml.dump(data, stream, allow_unicode=True)
data1 = stream.getvalue()
stream = BytesIO()
yaml.dump(data, stream, encoding='utf-16-le', allow_unicode=True)
data2 = stream.getvalue().decode('utf-16-le')[1:]
with open(filename, 'w', encoding='utf-16-le') as stream:
yaml.dump(data, stream, allow_unicode=True)
with open(filename, 'r', encoding='utf-16-le') as fp0:
data3 = fp0.read()
with open(filename, 'wb') as stream:
yaml.dump(data, stream, encoding='utf-8', allow_unicode=True)
with open(filename, 'r', encoding='utf-8') as fp0:
data4 = fp0.read()
assert data1 == data2, (data1, data2)
assert data1 == data3, (data1, data3)
assert data1 == data4, (data1, data4)
finally:
if os.path.exists(filename):
os.unlink(filename)
def serialize_all(nodes, stream=None, Dumper=Dumper,
canonical=None, indent=None, width=None,
allow_unicode=None, line_break=None,
encoding=enc, explicit_start=None, explicit_end=None,
version=None, tags=None):
# type: (Any, StreamType, Any, Any, Union[None, int], Union[None, int], bool, Any, Any, Union[None, bool], Union[None, bool], VersionType, Any) -> Any # NOQA
"""
Serialize a sequence of representation trees into a YAML stream.
If stream is None, return the produced string instead.
"""
getvalue = None
if stream is None:
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
allow_unicode=allow_unicode, line_break=line_break,
encoding=encoding, version=version, tags=tags,
explicit_start=explicit_start, explicit_end=explicit_end)
try:
dumper._serializer.open()
for node in nodes:
dumper.serialize(node)
dumper._serializer.close()
finally:
try:
dumper._emitter.dispose()
except AttributeError:
raise
dumper.dispose() # cyaml
if getvalue is not None:
return getvalue()
def test_unicode_input_errors(unicode_filename, verbose=False):
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
for input in [
data.encode('latin1', 'ignore'),
data.encode('utf-16-be'),
data.encode('utf-16-le'),
codecs.BOM_UTF8 + data.encode('utf-16-be'),
codecs.BOM_UTF16_BE + data.encode('utf-16-le'),
codecs.BOM_UTF16_LE + data.encode('utf-8') + b'!',
]:
try:
yaml.load(input)
except yaml.YAMLError as exc:
if verbose:
print(exc)
else:
raise AssertionError('expected an exception')
try:
yaml.load(BytesIO(input) if PY3 else StringIO(input))
except yaml.YAMLError as exc:
if verbose:
print(exc)
else:
raise AssertionError('expected an exception')
def dump_all(documents,
stream=None,
Dumper=Dumper,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=enc,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None):
"""
Serialize a sequence of Python objects into a YAML stream.
If stream is None, return the produced string instead.
"""
getvalue = None
if top_level_colon_align is True:
top_level_colon_align = max([len(str(x)) for x in documents[0]])
if stream is None:
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
dumper = Dumper(
stream,
default_style=default_style,
default_flow_style=default_flow_style,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
block_seq_indent=block_seq_indent,
top_level_colon_align=top_level_colon_align,
prefix_colon=prefix_colon,
)
try:
dumper.open()
for data in documents:
dumper.represent(data)
dumper.close()
finally:
dumper.dispose()
if getvalue:
return getvalue()
def test_unicode_output(unicode_filename, verbose=False):
yaml = YAML(typ='safe', pure=True)
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
value = ' '.join(data.split())
for allow_unicode in [False, True]:
data1 = yaml.dump(value, allow_unicode=allow_unicode)
for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']:
stream = StringIO()
yaml.dump(value,
stream,
encoding=encoding,
allow_unicode=allow_unicode)
data2 = stream.getvalue()
data3 = yaml.dump(value,
encoding=encoding,
allow_unicode=allow_unicode)
if encoding is not None:
assert isinstance(data3, bytes)
data3 = data3.decode(encoding)
stream = BytesIO()
if encoding is None:
try:
yaml.dump(value,
stream,
encoding=encoding,
allow_unicode=allow_unicode)
except TypeError as exc:
if verbose:
print(exc)
data4 = None
else:
raise AssertionError('expected an exception')
else:
yaml.dump(value,
stream,
encoding=encoding,
allow_unicode=allow_unicode)
data4 = stream.getvalue()
if verbose:
print('BYTES:', data4[:50])
data4 = data4.decode(encoding)
for copy in [data1, data2, data3, data4]:
if copy is None:
continue
assert isinstance(copy, str)
if allow_unicode:
try:
copy[4:].encode('ascii')
except UnicodeEncodeError as exc:
if verbose:
print(exc)
else:
raise AssertionError('expected an exception')
else:
copy[4:].encode('ascii')
assert isinstance(data1, str), (type(data1), encoding)
assert isinstance(data2, str), (type(data2), encoding)
def from_object(self, obj):
"""
:param obj: The object to serialize
:return: an open file-like object for streaming the serialized
bytes. Any file cursors should be set to the beginning
of the data (ala seek to the beginning).
"""
pruned_dict = self.conversion_policy.convert_from_object(obj)
# See if YAML should be pretty or not
yaml = YAML(typ='safe', pure=True)
if self.conversion_policy.is_pretty():
yaml.default_flow_style = False
yaml.indent(mapping=4, sequence=6, offset=3)
# Now convert the pruned dictionary to YAML
fileobj = BytesIO()
yaml.dump(pruned_dict, fileobj)
fileobj.seek(0, os.SEEK_SET)
return fileobj
def dump_all(self, documents, stream, _kw=enforce, transform=None):
# type: (Any, StreamType, Any, Any) -> Any
"""
Serialize a sequence of Python objects into a YAML stream.
"""
if not hasattr(stream, 'write') and hasattr(stream, 'open'):
# pathlib.Path() instance
with stream.open('w') as fp: # type: ignore
return self.dump_all(documents, fp, _kw, transform=transform)
if _kw is not enforce:
raise TypeError(
"{}.dump(_all) takes two positional argument but at least "
"three were given ({!r})".format(self.__class__.__name__, _kw))
# The stream should have the methods `write` and possibly `flush`.
if self.top_level_colon_align is True:
tlca = max([len(str(x)) for x in documents[0]]) # type: Any
else:
tlca = self.top_level_colon_align
if transform is not None:
fstream = stream
if self.encoding is None:
stream = StringIO()
else:
stream = BytesIO()
serializer, representer, emitter = \
self.get_serializer_representer_emitter(stream, tlca)
try:
self.serializer.open()
for data in documents:
try:
self.representer.represent(data)
except AttributeError:
# print(dir(dumper._representer))
raise
self.serializer.close()
finally:
try:
self.emitter.dispose()
except AttributeError:
raise
# self.dumper.dispose() # cyaml
delattr(self, "_serializer")
delattr(self, "_emitter")
if transform:
val = stream.getvalue() # type: ignore
if self.encoding:
val = val.decode(self.encoding)
if fstream is None:
transform(val)
else:
fstream.write(transform(val))
return None
def dump_all(documents, stream=None, Dumper=Dumper,
default_style=None, default_flow_style=None,
canonical=None, indent=None, width=None,
allow_unicode=None, line_break=None,
encoding=enc, explicit_start=None, explicit_end=None,
version=None, tags=None, block_seq_indent=None,
top_level_colon_align=None, prefix_colon=None):
# type: (Any, StreamType, Any, Any, Any, bool, Union[None, int], Union[None, int], bool, Any, Any, Union[None, bool], Union[None, bool], Any, Any, Any, Any, Any) -> Union[None, str] # NOQA
"""
Serialize a sequence of Python objects into a YAML stream.
If stream is None, return the produced string instead.
"""
getvalue = None
if top_level_colon_align is True:
top_level_colon_align = max([len(str(x)) for x in documents[0]])
if stream is None:
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
dumper = Dumper(stream, default_style=default_style,
default_flow_style=default_flow_style,
canonical=canonical, indent=indent, width=width,
allow_unicode=allow_unicode, line_break=line_break,
encoding=encoding, explicit_start=explicit_start,
explicit_end=explicit_end, version=version,
tags=tags, block_seq_indent=block_seq_indent,
top_level_colon_align=top_level_colon_align, prefix_colon=prefix_colon,
)
try:
dumper._serializer.open()
for data in documents:
try:
dumper._representer.represent(data)
except AttributeError:
# print(dir(dumper._representer))
raise
dumper._serializer.close()
finally:
try:
dumper._emitter.dispose()
except AttributeError:
raise
dumper.dispose() # cyaml
if getvalue is not None:
return getvalue()
return None
def dump_all(self, documents, stream=None):
# type: (Any, StreamType) -> Any
"""
Serialize a sequence of Python objects into a YAML stream.
If stream is None, return the produced string instead.
"""
# The stream should have the methods `write` and possibly `flush`.
if not hasattr(stream, 'write') and hasattr(stream, 'open'):
# pathlib.Path() instance
with stream.open('w') as fp: # type: ignore
return self.dump_all(documents, fp)
getvalue = None
if self.top_level_colon_align is True:
tlca = max([len(str(x)) for x in documents[0]]) # type: Any
else:
tlca = self.top_level_colon_align
if stream is None:
if self.encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
serializer, representer, emitter = \
self.get_serializer_representer_emitter(stream, tlca)
try:
self.serializer.open()
for data in documents:
try:
self.representer.represent(data)
except AttributeError:
# print(dir(dumper._representer))
raise
self.serializer.close()
finally:
try:
self.emitter.dispose()
except AttributeError:
raise
# self.dumper.dispose() # cyaml
delattr(self, "_serializer")
delattr(self, "_emitter")
if getvalue is not None:
return getvalue()
return None
def serialize_all(nodes,
stream=None,
Dumper=Dumper,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=enc,
explicit_start=None,
explicit_end=None,
version=None,
tags=None):
"""
Serialize a sequence of representation trees into a YAML stream.
If stream is None, return the produced string instead.
"""
getvalue = None
if stream is None:
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
dumper = Dumper(stream,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
encoding=encoding,
version=version,
tags=tags,
explicit_start=explicit_start,
explicit_end=explicit_end)
try:
dumper.open()
for node in nodes:
dumper.serialize(node)
dumper.close()
finally:
dumper.dispose()
if getvalue:
return getvalue()
def test_unicode_input(unicode_filename, verbose=False):
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
value = ' '.join(data.split())
output = yaml.load(data)
assert output == value, (output, value)
output = yaml.load(StringIO(data))
assert output == value, (output, value)
for input in [
data.encode('utf-8'),
codecs.BOM_UTF8 + data.encode('utf-8'),
codecs.BOM_UTF16_BE + data.encode('utf-16-be'),
codecs.BOM_UTF16_LE + data.encode('utf-16-le'),
]:
if verbose:
print('INPUT:', repr(input[:10]), '...')
output = yaml.load(input)
assert output == value, (output, value)
output = yaml.load(BytesIO(input))
assert output == value, (output, value)
def test_unicode_transfer(unicode_filename, verbose=False):
yaml = YAML(typ='safe', pure=True)
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']:
input = data
if encoding is not None:
input = ('\ufeff' + input).encode(encoding)
output1 = yaml.emit(yaml.parse(input), allow_unicode=True)
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
yaml.emit(yaml.parse(input), stream, allow_unicode=True)
output2 = stream.getvalue()
assert isinstance(output1, str), (type(output1), encoding)
if encoding is None:
assert isinstance(output2, str), (type(output1), encoding)
else:
assert isinstance(output2, bytes), (type(output1), encoding)
output2.decode(encoding)