def _test_catalog():
catalog = SymbolTableCatalog()
foo_1 = shared_symbol_table(u'foo', 1, [u'a', u'b'])
foo_3 = shared_symbol_table(u'foo', 3, [u'c', u'd', u'e', u'f'], imports=[foo_1])
bar_1 = shared_symbol_table(u'bar', 1, [u'x', u'y', u'z'])
zoo_4 = shared_symbol_table(u'zoo', 4, [u'm', u'n', u'o'])
for table in [foo_1, foo_3, bar_1, zoo_4, _SHADOW_ION_TABLE]:
catalog.register(table)
return catalog
def _test_catalog():
catalog = SymbolTableCatalog()
foo_1 = shared_symbol_table(u'foo', 1, [u'a', u'b'])
foo_3 = shared_symbol_table(u'foo',
3, [u'c', u'd', u'e', u'f'],
imports=[foo_1])
bar_1 = shared_symbol_table(u'bar', 1, [u'x', u'y', u'z'])
zoo_4 = shared_symbol_table(u'zoo', 4, [u'm', u'n', u'o'])
for table in [foo_1, foo_3, bar_1, zoo_4, _SHADOW_ION_TABLE]:
catalog.register(table)
return catalog
def write_with_shared_symbol_table_events():
structs = get_csv_structs()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
data = BytesIO()
writer = blocking_writer(binary_writer(imports=(table, )), data)
for struct in structs:
writer.send(IonEvent(IonEventType.CONTAINER_START, IonType.STRUCT))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.INT,
field_name=u'id',
value=struct[u'id']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.STRING,
field_name=u'type',
value=struct[u'type']))
writer.send(
IonEvent(IonEventType.SCALAR,
IonType.BOOL,
field_name=u'state',
value=struct[u'state']))
writer.send(IonEvent(IonEventType.CONTAINER_END))
writer.send(ION_STREAM_END_EVENT)
return data.getvalue()
def test_read_with_shared_symbol_table_simpleion():
# http://amzn.github.io/ion-docs/guides/cookbook.html#using-a-shared-symbol-table
data = write_with_shared_symbol_table_simpleion()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
catalog = SymbolTableCatalog()
catalog.register(table)
values = simpleion.loads(data, catalog=catalog, single_value=False)
assert values[2][u'id'] == 3
def write_with_shared_symbol_table_simpleion():
structs = get_csv_structs()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
data = simpleion.dumps(structs, imports=(table, ), sequence_as_stream=True)
# This byte literal is included in the examples.
assert data == b'\xe0\x01\x00\xea' \
b'\xee\xa4\x81\x83\xde\xa0\x86\xbe\x9b\xde\x99\x84\x8e\x90' \
b'test.csv.columns\x85!\x01\x88!\x03\x87\xb0\xde\x8a\x8a!' \
b'\x01\x8b\x83foo\x8c\x10\xde\x8a\x8a!\x02\x8b\x83bar\x8c' \
b'\x11\xde\x8a\x8a!\x03\x8b\x83baz\x8c\x11'
return data
def test_read_with_shared_symbol_table_events():
# http://amzn.github.io/ion-docs/guides/cookbook.html#using-a-shared-symbol-table
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
catalog = SymbolTableCatalog()
catalog.register(table)
data = BytesIO(write_with_shared_symbol_table_simpleion())
reader = blocking_reader(managed_reader(binary_reader(), catalog=catalog),
data)
# Position the reader at the first struct.
reader.send(NEXT_EVENT)
# Skip over the struct.
reader.send(SKIP_EVENT)
# Position the reader at the second struct.
reader.send(NEXT_EVENT)
# Skip over the struct.
reader.send(SKIP_EVENT)
# Position the reader at the third struct.
event = reader.send(NEXT_EVENT)
assert event.ion_type == IonType.STRUCT
# Step into the struct
event = reader.send(NEXT_EVENT)
assert u'id' == event.field_name.text
assert 3 == event.value
yield e_end_list()
if symbols is not None:
yield e_start_list(field_name=(token(TEXT_SYMBOLS)))
for symbol_text in symbols:
yield e_string(symbol_text)
yield e_end_list()
yield e_end_struct()
_SHADOW_ION_TEXTS = [
TEXT_ION_1_0, TEXT_ION_SYMBOL_TABLE, TEXT_NAME, TEXT_VERSION, TEXT_MAX_ID,
TEXT_IMPORTS, TEXT_SYMBOLS
]
_SHADOW_ION_TABLE = shared_symbol_table(u'shadow_ion', 1, _SHADOW_ION_TEXTS)
_SHADOW_ION_DESC = _desc(_SHADOW_ION_TABLE.name, _SHADOW_ION_TABLE.version,
_SHADOW_ION_TABLE.max_id)
def _test_catalog():
catalog = SymbolTableCatalog()
foo_1 = shared_symbol_table(u'foo', 1, [u'a', u'b'])
foo_3 = shared_symbol_table(u'foo',
3, [u'c', u'd', u'e', u'f'],
imports=[foo_1])
bar_1 = shared_symbol_table(u'bar', 1, [u'x', u'y', u'z'])
zoo_4 = shared_symbol_table(u'zoo', 4, [u'm', u'n', u'o'])
for table in [foo_1, foo_3, bar_1, zoo_4, _SHADOW_ION_TABLE]:
catalog.register(table)
def test_shared_symbls_malformed(p):
with pytest.raises(p.exc):
symbols.shared_symbol_table(p.name, p.version, p.symbols)
_P(name=None, version=None, symbols=[]),
_P(name=None, version=1, symbols=[]),
_P(name=u'my_shared', version=None, symbols=[]),
_P(name=u'my_shared', version=0, symbols=[]),
_P(name=u'my_shared', version=-1, symbols=[]),
_P(name=b'my_shared', version=1, symbols=[], exc=TypeError),
_P(name=u'my_shared', version=1, symbols=[b'a'], exc=TypeError),
)
def test_shared_symbls_malformed(p):
with pytest.raises(p.exc):
symbols.shared_symbol_table(p.name, p.version, p.symbols)
FOO_TEXTS = (u'a', u'b', u'c')
FOO_TABLE = symbols.shared_symbol_table(name=u'foo',
version=1,
symbols=FOO_TEXTS)
BAR_NEW_TEXTS = (u'c', u'd', u'e')
BAR_TEXTS = FOO_TEXTS + BAR_NEW_TEXTS
BAR_TABLE = symbols.shared_symbol_table(name=u'bar',
version=2,
symbols=BAR_NEW_TEXTS,
imports=(FOO_TABLE, ))
PLACEHOLDER = symbols.placeholder_symbol_table(u'placeholder', 1, 10)
PLACEHOLDER_TEXTS = tuple(itertools.repeat(None, 10))
SUB_SOURCE_LESS_SYMBOLS = symbols.substitute_symbol_table(BAR_TABLE, 3, 10)
SUB_SOURCE_LESS_TEXTS = tuple(
itertools.chain(BAR_TEXTS, itertools.repeat(None, 4)))
yield e_end_struct()
_SHADOW_ION_TEXTS = [
TEXT_ION_1_0,
TEXT_ION_SYMBOL_TABLE,
TEXT_NAME,
TEXT_VERSION,
TEXT_MAX_ID,
TEXT_IMPORTS,
TEXT_SYMBOLS
]
_SHADOW_ION_TABLE = shared_symbol_table(
u'shadow_ion',
1,
_SHADOW_ION_TEXTS
)
_SHADOW_ION_DESC = _desc(
_SHADOW_ION_TABLE.name, _SHADOW_ION_TABLE.version, _SHADOW_ION_TABLE.max_id
)
def _test_catalog():
catalog = SymbolTableCatalog()
foo_1 = shared_symbol_table(u'foo', 1, [u'a', u'b'])
foo_3 = shared_symbol_table(u'foo', 3, [u'c', u'd', u'e', u'f'], imports=[foo_1])
bar_1 = shared_symbol_table(u'bar', 1, [u'x', u'y', u'z'])
zoo_4 = shared_symbol_table(u'zoo', 4, [u'm', u'n', u'o'])
for table in [foo_1, foo_3, bar_1, zoo_4, _SHADOW_ION_TABLE]:
from __future__ import print_function
from pytest import raises
from tests import parametrize, is_exception
import amazon.ion.symbols as symbols
from amazon.ion.exceptions import CannotSubstituteTable
from amazon.ion.util import record
FOO_1_TEXTS = (u'aa', u'bb', u'cc')
FOO_1_TABLE = symbols.shared_symbol_table(
name=u'foo',
version=1,
symbols=FOO_1_TEXTS
)
# Note the one symbol symbol overlap.
FOO_2_TEXTS = (u'cc', u'dd', u'ee')
FOO_2_TABLE = symbols.shared_symbol_table(
name=u'foo',
version=2,
imports=[FOO_1_TABLE],
symbols=FOO_2_TEXTS
)
# Note the gap in version.
FOO_4_TEXTS = (u'ff', u'gg', u'hh')
FOO_4_TABLE = symbols.shared_symbol_table(
def test_shared_symbls_malformed(p):
with pytest.raises(p.exc):
symbols.shared_symbol_table(p.name, p.version, p.symbols)
_P(name=None, version=1, symbols=[]),
_P(name=u'my_shared', version=None, symbols=[]),
_P(name=u'my_shared', version=0, symbols=[]),
_P(name=u'my_shared', version=-1, symbols=[]),
_P(name=b'my_shared', version=1, symbols=[], exc=TypeError),
_P(name=u'my_shared', version=1, symbols=[b'a'], exc=TypeError),
)
def test_shared_symbls_malformed(p):
with pytest.raises(p.exc):
symbols.shared_symbol_table(p.name, p.version, p.symbols)
FOO_TEXTS = (u'a', u'b', u'c')
FOO_TABLE = symbols.shared_symbol_table(
name=u'foo',
version=1,
symbols=FOO_TEXTS
)
BAR_NEW_TEXTS = (u'c', u'd', u'e')
BAR_TEXTS = FOO_TEXTS + BAR_NEW_TEXTS
BAR_TABLE = symbols.shared_symbol_table(
name=u'bar',
version=2,
symbols=BAR_NEW_TEXTS,
imports=(FOO_TABLE,)
)
PLACEHOLDER = symbols.placeholder_symbol_table(u'placeholder', 1, 10)
PLACEHOLDER_TEXTS = tuple(itertools.repeat(None, 10))
def table_ion_fmt(json, names):
syms = symbols.shared_symbol_table(None, symbols=map(str, names))
formatted = ion.dumps(json, binary=True, imports=[syms])
return formatted