def write_with_shared_symbol_table_events(): structs = get_csv_structs() table = shared_symbol_table(u'test.csv.columns', 1, (u'id', u'type', u'state')) data = BytesIO() writer = blocking_writer(binary_writer(imports=(table, )), data) for struct in structs: writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT)) writer.send( IonEvent(IonEventType.SCALAR, IonType.INT, field_name=u'id', value=struct[u'id'])) writer.send( IonEvent(IonEventType.SCALAR, IonType.STRING, field_name=u'type', value=struct[u'type'])) writer.send( IonEvent(IonEventType.SCALAR, IonType.BOOL, field_name=u'state', value=struct[u'state'])) writer.send(IonEvent(IonEventType.CONTAINER_END)) writer.send(ION_STREAM_END_EVENT) return data.getvalue()
def test_convert_csv_events(): # http://amzn.github.io/ion-docs/guides/cookbook.html#converting-non-hierarchical-data-to-ion structs = get_csv_structs() ion = BytesIO() writer = blocking_writer(binary_writer(), ion) for struct in structs: writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT)) writer.send( IonEvent(IonEventType.SCALAR, IonType.INT, field_name=u'id', value=struct[u'id'])) writer.send( IonEvent(IonEventType.SCALAR, IonType.STRING, field_name=u'type', value=struct[u'type'])) writer.send( IonEvent(IonEventType.SCALAR, IonType.BOOL, field_name=u'state', value=struct[u'state'])) writer.send(IonEvent(IonEventType.CONTAINER_END)) writer.send(ION_STREAM_END_EVENT) assert b'\xe0\x01\x00\xea\xee\x95\x81\x83\xde\x91\x87\xbe\x8e\x82id\x84type\x85state\xde\x8a\x8a!' \ b'\x01\x8b\x83foo\x8c\x10\xde\x8a\x8a!\x02\x8b\x83bar\x8c\x11\xde\x8a\x8a!\x03\x8b\x83baz\x8c\x11' \ == ion.getvalue()
def test_one_ivm_per_stream(): out, writer = new_writer() # Explicitly write the IVM. writer.send(IonEvent(IonEventType.VERSION_MARKER)) # If the IVM not explicitly written, this would write one. writer.send(IonEvent(IonEventType.SCALAR, IonType.NULL)) finish(writer) buf = out.getvalue() assert_ivm(buf)
def test_reuse_after_flush(): out, writer = new_writer() writer.send(IonEvent(IonEventType.SCALAR, IonType.BOOL, True)) finish(writer) length = len(out.getvalue()) writer.send(IonEvent(IonEventType.SCALAR, IonType.BOOL, False)) finish(writer) buf = out.getvalue() assert length != len(buf) assert_ivm(buf, 2)
def generate_containers(containers_map, force_stream_end=False): for ion_type, container in six.iteritems(containers_map): for container_value_events, expected in container: start_event = IonEvent(IonEventType.CONTAINER_START, ion_type) end_event = IonEvent(IonEventType.CONTAINER_END, ion_type) events = (start_event, ) + container_value_events + (end_event, ) if force_stream_end: events += _STREAM_END_EVENT yield WriterParameter( desc='CONTAINER %s - %r' % (ion_type.name, expected), events=events, expected=expected, )
def generate_scalars_binary(scalars_map, preceding_symbols=0): for ion_type, values in six.iteritems(scalars_map): for native, expected in values: native_expected = expected has_symbols = False if native is None: # An un-adorned 'None' doesn't contain enough information to determine its Ion type native_expected = b'\x0f' elif ion_type is IonType.CLOB: # All six.binary_type are treated as BLOBs unless wrapped by an _IonNature tid = six.byte2int( expected ) + 0x10 # increment upper nibble for clob -> blob; keep lower nibble native_expected = bytearray([tid]) + expected[1:] elif ion_type is IonType.SYMBOL and native is not None: has_symbols = True elif ion_type is IonType.STRING: # Encode all strings as symbols too. symbol_expected = _serialize_symbol( IonEvent(IonEventType.SCALAR, IonType.SYMBOL, SymbolToken(None, 10 + preceding_symbols))) yield _Parameter(IonType.SYMBOL.name + ' ' + native, IonPyText.from_value(IonType.SYMBOL, native), symbol_expected, True) yield _Parameter('%s %s' % (ion_type.name, native), native, native_expected, has_symbols) wrapper = _FROM_ION_TYPE[ion_type].from_value(ion_type, native) yield _Parameter(repr(wrapper), wrapper, expected, has_symbols)
def test_implicit_ivm(): out, writer = new_writer() writer.send(IonEvent(IonEventType.SCALAR, IonType.NULL)) # This implicitly writes the IVM. finish(writer) buf = out.getvalue() assert_ivm(buf)
def _scalar_p(ion_type, value, expected, force_stream_end): events = (IonEvent(IonEventType.SCALAR, ion_type, value), ) if force_stream_end: events += _STREAM_END_EVENT return WriterParameter( desc='SCALAR %s - %r - %r' % (ion_type.name, value, expected), events=events, expected=expected, )
def test_finish_resets_buffers(): out, writer = new_writer() writer.send(IonEvent(IonEventType.SCALAR, IonType.STRING, u'foo')) finish(writer) buf = out.getvalue() assert len(buf) != 0 out.truncate(0) finish(writer) buf = out.getvalue() assert len(buf) == 0
def test_write_numeric_with_annotation_events(): # http://amzn.github.io/ion-docs/guides/cookbook.html#reading-numeric-types event = IonEvent(IonEventType.SCALAR, IonType.FLOAT, annotations=(u'abc', ), value=123.0) data = BytesIO() writer = blocking_writer(text_writer(), data) writer.send(event) writer.send(ION_STREAM_END_EVENT) assert u'abc::123.0e0' == data.getvalue().decode(u'utf-8')
def test_no_fieldname_in_hash(test_data): """ This test verifies a hash_reader/writer that receives field events but did not receive the preceeding "begin struct" event DOES NOT include the field_name as part of the hash. """ reader = binary_reader_over(test_data.ion_str) events = consume(reader) buf = BytesIO() writer = blocking_writer(raw_writer(), buf) writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT)) hw = hash_writer(writer, hash_function_provider("identity")) for e in events[:-1]: field_name = e.field_name if e.depth == 0: field_name = SymbolToken("field_name", None) new_event = IonEvent(e.event_type, e.ion_type, e.value, field_name, e.annotations, e.depth + 1) hw.send(new_event) writer.send(IonEvent(IonEventType.CONTAINER_END, IonType.STRUCT)) writer.send(events[-1]) # send the final event (which should be a STREAM_END event) output = buf.getvalue() hw_digest = hw.send(HashEvent.DIGEST) assert hw_digest == test_data.expected_digest reader = binary_reader_over(output) hr_digest = b'' while True: event = reader.send(NEXT_EVENT) if event.event_type == IonEventType.CONTAINER_START: hr = hash_reader(reader, hash_function_provider("identity")) for i in range(0, len(events) - 1): e = hr.send(NEXT_EVENT) hr_digest = hr.send(HashEvent.DIGEST) if event.event_type == IonEventType.STREAM_END: break assert hr_digest == test_data.expected_digest
def test_writing_events_blocking(): # http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data data = BytesIO() writer = blocking_writer(binary_writer(), data) event_type = writer.send( IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT)) # The value is not complete, so more events are required. assert event_type == WriteEventType.NEEDS_INPUT event_type = writer.send( IonEvent(IonEventType.SCALAR, IonType.STRING, field_name=u'hello', value=u'world')) # The value is not complete, so more events are required. assert event_type == WriteEventType.NEEDS_INPUT event_type = writer.send(IonEvent(IonEventType.CONTAINER_END)) # The value is not complete, so more events are required. assert event_type == WriteEventType.NEEDS_INPUT event_type = writer.send(ION_STREAM_END_EVENT) # The end of the stream was signaled, so the data has been flushed. assert event_type == WriteEventType.COMPLETE assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data.getvalue( )
def test_writing_events_non_blocking(): # http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data def drain_data(incremental_event): incremental_data = b'' while incremental_event.type == WriteEventType.HAS_PENDING: incremental_data += incremental_event.data incremental_event = writer.send(None) return incremental_data writer = binary_writer() event = writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT)) data = drain_data(event) event = writer.send( IonEvent(IonEventType.SCALAR, IonType.STRING, field_name=u'hello', value=u'world')) data += drain_data(event) event = writer.send(IonEvent(IonEventType.CONTAINER_END)) data += drain_data(event) event = writer.send(ION_STREAM_END_EVENT) data += drain_data(event) assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == data
def _f(algorithm): buf.seek(0) reader = ion_reader.blocking_reader( managed_reader(reader_provider(), None), buf) writer = hash_writer( blocking_writer(raw_writer(), BytesIO()), hash_function_provider(algorithm, _actual_updates, _actual_digests)) _consume(reader, writer) digest = writer.send(HashEvent.DIGEST) writer.send(IonEvent(IonEventType.STREAM_END)) return digest
def test_import_lst_fails(): lst = SymbolTable(LOCAL_TABLE_TYPE, [u'foo']) out, writer = new_writer([lst]) with raises(IonException): # TODO fail earlier? writer.send(IonEvent(IonEventType.SCALAR, IonType.STRING, u'bar'))
def finish(writer): writer.send(IonEvent(IonEventType.STREAM_END))