def write_with_shared_symbol_table_events():
structs = get_csv_structs()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
data = BytesIO()
writer = blocking_writer(binary_writer(imports=(table, )), data)
for struct in structs:
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.INT,
field_name=u'id',
value=struct[u'id']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'type',
value=struct[u'type']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.BOOL,
field_name=u'state',
value=struct[u'state']))
writer.send(IonEvent(IonEventType.CONTAINER_END))
writer.send(ION_STREAM_END_EVENT)
return data.getvalue()
def test_convert_csv_events():
# http://amzn.github.io/ion-docs/guides/cookbook.html#converting-non-hierarchical-data-to-ion
structs = get_csv_structs()
ion = BytesIO()
writer = blocking_writer(binary_writer(), ion)
for struct in structs:
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.INT,
field_name=u'id',
value=struct[u'id']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'type',
value=struct[u'type']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.BOOL,
field_name=u'state',
value=struct[u'state']))
writer.send(IonEvent(IonEventType.CONTAINER_END))
writer.send(ION_STREAM_END_EVENT)
assert b'\xe0\x01\x00\xea\xee\x95\x81\x83\xde\x91\x87\xbe\x8e\x82id\x84type\x85state\xde\x8a\x8a!' \
b'\x01\x8b\x83foo\x8c\x10\xde\x8a\x8a!\x02\x8b\x83bar\x8c\x11\xde\x8a\x8a!\x03\x8b\x83baz\x8c\x11' \
== ion.getvalue()
def test_writing_events_blocking():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
data = BytesIO()
writer = blocking_writer(binary_writer(), data)
event_type = writer.send(
IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'hello',
value=u'world'))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(IonEvent(IonEventType.CONTAINER_END))
# The value is not complete, so more events are required.
assert event_type == WriteEventType.NEEDS_INPUT
event_type = writer.send(ION_STREAM_END_EVENT)
# The end of the stream was signaled, so the data has been flushed.
assert event_type == WriteEventType.COMPLETE
assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data.getvalue(
)
def test_writing_events_non_blocking():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
def drain_data(incremental_event):
incremental_data = b''
while incremental_event.type == WriteEventType.HAS_PENDING:
incremental_data += incremental_event.data
incremental_event = writer.send(None)
return incremental_data
writer = binary_writer()
event = writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
data = drain_data(event)
event = writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'hello',
value=u'world'))
data += drain_data(event)
event = writer.send(IonEvent(IonEventType.CONTAINER_END))
data += drain_data(event)
event = writer.send(ION_STREAM_END_EVENT)
data += drain_data(event)
assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data
def ion_hash(self, algorithm=None, hash_function_provider=None):
"""Given an algorithm or hash_function_provider, computes the Ion hash
of this value.
Args:
algorithm:
A string corresponding to the name of a hash algorithm supported
by the `hashlib` module.
hash_function_provider:
A function that returns a new ``IonHasher`` instance when called.
Note that multiple ``IonHasher`` instances may be required to hash a single value
(depending on the type of the Ion value).
Returns:
`bytes` that represent the Ion hash of this value for the specified algorithm
or hash_function_provider.
"""
if algorithm is None and hash_function_provider is None:
raise Exception(
"Either 'algorithm' or 'hash_function_provider' must be specified")
if algorithm is not None and hash_function_provider is not None:
raise Exception(
"Either 'algorithm' or 'hash_function_provider' must be specified, not both"
)
if algorithm is not None:
hfp = hashlib_hash_function_provider(algorithm)
else:
hfp = hash_function_provider
hw = hash_writer(blocking_writer(binary_writer(), BytesIO()), hfp)
_dump(self, hw, _FROM_TYPE)
hw.send(ION_STREAM_END_EVENT)
return hw.send(HashEvent.DIGEST)
def _f():
if type == "binary":
return binary_writer()
elif type == "text":
return raw_writer()
def new_writer(imports=None):
out = BytesIO()
return out, blocking_writer(binary_writer(imports), out)
def new_writer(imports=None):
out = BytesIO()
return out, blocking_writer(binary_writer(imports), out)