def write_with_shared_symbol_table_events():
structs = get_csv_structs()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
data = BytesIO()
writer = blocking_writer(binary_writer(imports=(table, )), data)
for struct in structs:
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.INT,
field_name=u'id',
value=struct[u'id']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'type',
value=struct[u'type']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.BOOL,
field_name=u'state',
value=struct[u'state']))
writer.send(IonEvent(IonEventType.CONTAINER_END))
writer.send(ION_STREAM_END_EVENT)
return data.getvalue()
def test_convert_csv_events():
# http://amzn.github.io/ion-docs/guides/cookbook.html#converting-non-hierarchical-data-to-ion
structs = get_csv_structs()
ion = BytesIO()
writer = blocking_writer(binary_writer(), ion)
for struct in structs:
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.INT,
field_name=u'id',
value=struct[u'id']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'type',
value=struct[u'type']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.BOOL,
field_name=u'state',
value=struct[u'state']))
writer.send(IonEvent(IonEventType.CONTAINER_END))
writer.send(ION_STREAM_END_EVENT)
assert b'\xe0\x01\x00\xea\xee\x95\x81\x83\xde\x91\x87\xbe\x8e\x82id\x84type\x85state\xde\x8a\x8a!' \
b'\x01\x8b\x83foo\x8c\x10\xde\x8a\x8a!\x02\x8b\x83bar\x8c\x11\xde\x8a\x8a!\x03\x8b\x83baz\x8c\x11' \
== ion.getvalue()
def test_one_ivm_per_stream():
out, writer = new_writer()
# Explicitly write the IVM.
writer.send(IonEvent(IonEventType.VERSION_MARKER))
# If the IVM not explicitly written, this would write one.
writer.send(IonEvent(IonEventType.SCALAR, IonType.NULL))
finish(writer)
buf = out.getvalue()
assert_ivm(buf)
def test_reuse_after_flush():
out, writer = new_writer()
writer.send(IonEvent(IonEventType.SCALAR, IonType.BOOL, True))
finish(writer)
length = len(out.getvalue())
writer.send(IonEvent(IonEventType.SCALAR, IonType.BOOL, False))
finish(writer)
buf = out.getvalue()
assert length != len(buf)
assert_ivm(buf, 2)
def generate_containers(containers_map, force_stream_end=False):
for ion_type, container in six.iteritems(containers_map):
for container_value_events, expected in container:
start_event = IonEvent(IonEventType.CONTAINER_START, ion_type)
end_event = IonEvent(IonEventType.CONTAINER_END, ion_type)
events = (start_event, ) + container_value_events + (end_event, )
if force_stream_end:
events += _STREAM_END_EVENT
yield WriterParameter(
desc='CONTAINER %s - %r' % (ion_type.name, expected),
events=events,
expected=expected,
)
def generate_scalars_binary(scalars_map, preceding_symbols=0):
for ion_type, values in six.iteritems(scalars_map):
for native, expected in values:
native_expected = expected
has_symbols = False
if native is None:
# An un-adorned 'None' doesn't contain enough information to determine its Ion type
native_expected = b'\x0f'
elif ion_type is IonType.CLOB:
# All six.binary_type are treated as BLOBs unless wrapped by an _IonNature
tid = six.byte2int(
expected
) + 0x10 # increment upper nibble for clob -> blob; keep lower nibble
native_expected = bytearray([tid]) + expected[1:]
elif ion_type is IonType.SYMBOL and native is not None:
has_symbols = True
elif ion_type is IonType.STRING:
# Encode all strings as symbols too.
symbol_expected = _serialize_symbol(
IonEvent(IonEventType.SCALAR, IonType.SYMBOL,
SymbolToken(None, 10 + preceding_symbols)))
yield _Parameter(IonType.SYMBOL.name + ' ' + native,
IonPyText.from_value(IonType.SYMBOL, native),
symbol_expected, True)
yield _Parameter('%s %s' % (ion_type.name, native), native,
native_expected, has_symbols)
wrapper = _FROM_ION_TYPE[ion_type].from_value(ion_type, native)
yield _Parameter(repr(wrapper), wrapper, expected, has_symbols)
def test_implicit_ivm():
out, writer = new_writer()
writer.send(IonEvent(IonEventType.SCALAR,
IonType.NULL)) # This implicitly writes the IVM.
finish(writer)
buf = out.getvalue()
assert_ivm(buf)
def _scalar_p(ion_type, value, expected, force_stream_end):
events = (IonEvent(IonEventType.SCALAR, ion_type, value), )
if force_stream_end:
events += _STREAM_END_EVENT
return WriterParameter(
desc='SCALAR %s - %r - %r' % (ion_type.name, value, expected),
events=events,
expected=expected,
)
def test_finish_resets_buffers():
out, writer = new_writer()
writer.send(IonEvent(IonEventType.SCALAR, IonType.STRING, u'foo'))
finish(writer)
buf = out.getvalue()
assert len(buf) != 0
out.truncate(0)
finish(writer)
buf = out.getvalue()
assert len(buf) == 0
def test_write_numeric_with_annotation_events():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-numeric-types
event = IonEvent(IonEventType.SCALAR,
IonType.FLOAT,
annotations=(u'abc', ),
value=123.0)
data = BytesIO()
writer = blocking_writer(text_writer(), data)
writer.send(event)
writer.send(ION_STREAM_END_EVENT)
assert u'abc::123.0e0' == data.getvalue().decode(u'utf-8')
def test_no_fieldname_in_hash(test_data):
"""
This test verifies a hash_reader/writer that receives field events but did not
receive the preceeding "begin struct" event DOES NOT include the field_name as
part of the hash.
"""
reader = binary_reader_over(test_data.ion_str)
events = consume(reader)
buf = BytesIO()
writer = blocking_writer(raw_writer(), buf)
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
hw = hash_writer(writer, hash_function_provider("identity"))
for e in events[:-1]:
field_name = e.field_name
if e.depth == 0:
field_name = SymbolToken("field_name", None)
new_event = IonEvent(e.event_type, e.ion_type, e.value, field_name, e.annotations, e.depth + 1)
hw.send(new_event)
writer.send(IonEvent(IonEventType.CONTAINER_END, IonType.STRUCT))
writer.send(events[-1]) # send the final event (which should be a STREAM_END event)
output = buf.getvalue()
hw_digest = hw.send(HashEvent.DIGEST)
assert hw_digest == test_data.expected_digest
reader = binary_reader_over(output)
hr_digest = b''
while True:
event = reader.send(NEXT_EVENT)
if event.event_type == IonEventType.CONTAINER_START:
hr = hash_reader(reader, hash_function_provider("identity"))
for i in range(0, len(events) - 1):
e = hr.send(NEXT_EVENT)
hr_digest = hr.send(HashEvent.DIGEST)
if event.event_type == IonEventType.STREAM_END:
break
assert hr_digest == test_data.expected_digest
def test_writing_events_blocking():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
data = BytesIO()
writer = blocking_writer(binary_writer(), data)
event_type = writer.send(
IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'hello',
value=u'world'))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(IonEvent(IonEventType.CONTAINER_END))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(ION_STREAM_END_EVENT)
# The end of the stream was signaled, so the data has been flushed.
assert event_type == WriteEventType.COMPLETE
assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data.getvalue(
)
def test_writing_events_non_blocking():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
def drain_data(incremental_event):
incremental_data = b''
while incremental_event.type == WriteEventType.HAS_PENDING:
incremental_data += incremental_event.data
incremental_event = writer.send(None)
return incremental_data
writer = binary_writer()
event = writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
data = drain_data(event)
event = writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'hello',
value=u'world'))
data += drain_data(event)
event = writer.send(IonEvent(IonEventType.CONTAINER_END))
data += drain_data(event)
event = writer.send(ION_STREAM_END_EVENT)
data += drain_data(event)
assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data
def _f(algorithm):
buf.seek(0)
reader = ion_reader.blocking_reader(
managed_reader(reader_provider(), None), buf)
writer = hash_writer(
blocking_writer(raw_writer(), BytesIO()),
hash_function_provider(algorithm, _actual_updates,
_actual_digests))
_consume(reader, writer)
digest = writer.send(HashEvent.DIGEST)
writer.send(IonEvent(IonEventType.STREAM_END))
return digest
def test_import_lst_fails():
lst = SymbolTable(LOCAL_TABLE_TYPE, [u'foo'])
out, writer = new_writer([lst])
with raises(IonException): # TODO fail earlier?
writer.send(IonEvent(IonEventType.SCALAR, IonType.STRING, u'bar'))
def finish(writer):
writer.send(IonEvent(IonEventType.STREAM_END))
