def generate_tests_ion_hash_tests(base_dir, out_dir):
"""
Writes each test value in ion_hash_test.ion as a top-level value in new files
"<out_dir>/ion_hash_tests.ion" and "<out_dir>/ion_hash_tests.10n".
"""
with open(os.path.join(out_dir, "ion_hash_tests.ion"), "w") as text_file, \
open(os.path.join(out_dir, "ion_hash_tests.10n"), "wb") as binary_file:
with open(os.path.join(base_dir, "ion_hash_tests.ion")) as f:
ion_hash_tests = f.read()
tests = ion.loads(ion_hash_tests, single_value=False)
for test in tests:
if 'ion' in test:
test_text = ion.dumps(test['ion'],
binary=False,
omit_version_marker=True)
if not (
'$0' in test_text
): # https://github.com/amzn/ion-hash-test-driver/issues/1
text_file.write(test_text + "\n")
test_binary = ion.dumps(test['ion'], binary=True)
binary_file.write(test_binary)
if '10n' in test:
sexp_bytes = sexp_to_bytearray(test['10n'])
binary_file.write(sexp_bytes)
return [text_file.name, binary_file.name]
def test_query_table_enclosed_in_quotes(self):
# Given.
# Create Ion struct to insert.
ion_value = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE}))
query = "INSERT INTO {} ?".format(TABLE_NAME)
def execute_statement_with_parameter_and_return_count(
txn, query, parameter):
cursor = txn.execute_statement(query, parameter)
count = 0
for row in cursor:
count += 1
return count
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_with_parameter_and_return_count(
txn, query, ion_value))
self.assertEqual(1, count)
search_query = "SELECT VALUE {} FROM \"{}\" WHERE {} = ?".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
ion_string = loads(dumps(SINGLE_DOCUMENT_VALUE))
def execute_statement_and_return_value(txn, query, *parameters):
cursor = txn.execute_statement(query, *parameters)
return next(cursor)
# When.
value = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_value(
txn, search_query, ion_string))
# Then.
self.assertEqual(SINGLE_DOCUMENT_VALUE, value)
def test_dumps_omit_version_marker():
v = loads('5')
assert dumps(v, binary=False) == '$ion_1_0 5'
assert dumps(v, binary=False, omit_version_marker=True) == '5'
# verify no impact on binary output
assert dumps(v) == b'\xe0\x01\x00\xea\x21\x05'
assert dumps(v, omit_version_marker=True) == b'\xe0\x01\x00\xea\x21\x05'
def test_sexp():
if is_pypy:
return
value = (six.text_type("Ion"), 123)
ion_value = loads(dumps(loads(dumps(value, tuple_as_sexp=True))))
assert isinstance(ion_value, IonPyList) and ion_value.ion_type == IonType.SEXP
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '["Ion", 123]'
def test_struct_field():
# pass a dict through simpleion to get a reconstituted dict of Ion values.
struct_a = loads(dumps({u'dont_remember_my_name': 1}))
# copy the value of the "dont_remember_my_name" field to a new struct, which is also passed through simpleion
struct_b = {u'new_name': struct_a[u"dont_remember_my_name"]}
struct_c = loads(dumps(struct_b))
# The bug identified in ion-python#95 is that the name of the original field is somehow preserved.
# verify this no longer happens
assert u'dont_remember_my_name' not in struct_c
assert u'new_name' in struct_c
def fill_table():
# Create Ion structs to insert to populate table
parameter_1 = loads(dumps({COLUMN_NAME:
MULTIPLE_DOCUMENT_VALUE_1}))
parameter_2 = loads(dumps({COLUMN_NAME:
MULTIPLE_DOCUMENT_VALUE_2}))
parameter_3 = loads(dumps({COLUMN_NAME:
MULTIPLE_DOCUMENT_VALUE_3}))
query = "INSERT INTO {} <<?, ?, ?>>".format(TABLE_NAME)
self.qldb_driver.execute_lambda(lambda txn: txn.execute_statement(
query, parameter_1, parameter_2, parameter_3))
def test_delete_single_document(self):
# Given.
# Create Ion struct to insert.
ion_value = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE}))
query = "INSERT INTO {} ?".format(TABLE_NAME)
def execute_statement_with_parameter_and_return_count(
txn, query, parameter):
cursor = txn.execute_statement(query, parameter)
count = 0
for row in cursor:
count += 1
return count
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_with_parameter_and_return_count(
txn, query, ion_value))
self.assertEqual(1, count)
# When.
delete_query = "DELETE FROM {} WHERE {} = ?".format(
TABLE_NAME, COLUMN_NAME)
ion_string = loads(dumps(SINGLE_DOCUMENT_VALUE))
def execute_statement_and_return_count(txn, query, *parameters):
cursor = txn.execute_statement(query, *parameters)
count = 0
for row in cursor:
count += 1
return count
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_count(
txn, delete_query, ion_string))
self.assertEqual(1, count)
# Then.
def execute_count_statement_and_return_count(txn):
search_query = "SELECT COUNT(*) FROM {}".format(TABLE_NAME)
# This gives:
# {
# _1: 1
# }
cursor = txn.execute_statement(search_query)
return next(cursor)['_1']
count = self.qldb_driver.execute_lambda(
lambda txn: execute_count_statement_and_return_count(txn))
self.assertEqual(0, count)
def test_delete_all_documents(self):
# Given.
# Create Ion structs to insert.
parameter_1 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_1}))
parameter_2 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_2}))
query = "INSERT INTO {} <<?, ?>>".format(TABLE_NAME)
def execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2):
cursor = txn.execute_statement(query, parameter_1, parameter_2)
count = 0
for row in cursor:
count += 1
return count
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2))
self.assertEqual(2, count)
# When.
delete_query = "DELETE FROM {}".format(TABLE_NAME, COLUMN_NAME)
def execute_statement_and_return_count(txn, query):
cursor = txn.execute_statement(query)
count = 0
for row in cursor:
count += 1
return count
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_count(txn, delete_query))
self.assertEqual(2, count)
# Then.
def execute_count_statement_and_return_count(txn):
search_query = "SELECT COUNT(*) FROM {}".format(TABLE_NAME)
# This gives:
# {
# _1: 1
# }
cursor = txn.execute_statement(search_query)
return next(cursor)['_1']
count = self.qldb_driver.execute_lambda(
lambda txn: execute_count_statement_and_return_count(txn))
self.assertEqual(0, count)
def sparse_reads_data():
data = u'''
$ion_1_0
foo::{
quantity: 1
}
bar::{
name: "x",
id: 7
}
baz::{
items:["thing1", "thing2"]
}
foo::{
quantity: 19
}
bar::{
name: "y",
id: 8
}'''
data = simpleion.dumps(simpleion.loads(data, single_value=False),
sequence_as_stream=True)
# This byte literal is included in the examples.
assert data == b'\xe0\x01\x00\xea' \
b'\xee\xa5\x81\x83\xde\xa1\x87\xbe\x9e\x83foo\x88quantity\x83' \
b'bar\x82id\x83baz\x85items\xe7\x81\x8a\xde\x83\x8b!\x01\xea' \
b'\x81\x8c\xde\x86\x84\x81x\x8d!\x07\xee\x95\x81\x8e\xde\x91' \
b'\x8f\xbe\x8e\x86thing1\x86thing2\xe7\x81\x8a\xde\x83\x8b!' \
b'\x13\xea\x81\x8c\xde\x86\x84\x81y\x8d!\x08'
return data
def enough_inventory_registered_for_order(transaction_executor,
products_per_case, purchase_order_id,
batch_table_name):
container_order_quantity = get_value_from_documentid(
transaction_executor, Constants.PURCHASE_ORDER_TABLE_NAME,
purchase_order_id, "OrderQuantity")
container_order_quantity = int(container_order_quantity[0])
product_units_ordered = container_order_quantity * Constants.PALLETS_PER_CONTAINER * Constants.CASES_PER_PALLETE * int(
products_per_case)
statement = 'SELECT SUM(b.UnitsRemaining) as invrem FROM {} as b WHERE b.UnitsRemaining > 0'.format(
batch_table_name)
cursor = transaction_executor.execute_statement(statement)
inventory_remaining = list(map(lambda x: x.get('invrem'), cursor))
inventory_remaining = int(
dumps(inventory_remaining[0],
binary=False,
indent=' ',
omit_version_marker=True))
if inventory_remaining >= product_units_ordered:
# logger.info(" Enough inventory to pack the order: {}!".format(inventory_remaining))
return product_units_ordered
else:
# logger.info("Not enough inventory!")
return False
def test_convert_csv_simpleion():
# http://amzn.github.io/ion-docs/guides/cookbook.html#converting-non-hierarchical-data-to-ion
structs = get_csv_structs()
ion = simpleion.dumps(structs, sequence_as_stream=True)
assert b'\xe0\x01\x00\xea\xee\x95\x81\x83\xde\x91\x87\xbe\x8e\x82id\x84type\x85state\xde\x8a\x8a!' \
b'\x01\x8b\x83foo\x8c\x10\xde\x8a\x8a!\x02\x8b\x83bar\x8c\x11\xde\x8a\x8a!\x03\x8b\x83baz\x8c\x11' \
== ion
def _to_value_holder(parameter):
"""
Convert Python or Ion value to binary, and store in a value holder.
"""
parameter_binary = dumps(parameter)
value_holder = {'IonBinary': parameter_binary}
return value_holder
def test_blob():
if is_pypy:
return
ion_value = loads(dumps(b64encode("Ion".encode("ASCII")) if six.PY2 else bytes("Ion", "ASCII")))
assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.BLOB
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"SW9u"'
def test_timestamp():
if is_pypy:
return
ion_value = loads(dumps(datetime.datetime(2010, 6, 15, 3, 30, 45)))
assert isinstance(ion_value, IonPyTimestamp) and ion_value.ion_type == IonType.TIMESTAMP
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"2010-06-15 03:30:45"'
def _to_buffer(ion_test, binary):
if 'ion' in ion_test:
v = ion.dumps(ion_test['ion'], binary=binary)
if '10n' in ion_test:
v = bytearray(_IVM_BYTES)
for byte in ion_test['10n']:
v.append(byte)
if not binary:
value = ion.load(BytesIO(v))
v = ion.dumps(value, binary=False)
if binary:
return BytesIO(v)
else:
return StringIO(v)
def test_decimal_precision():
from decimal import localcontext
with localcontext() as ctx:
# ensure test executes with the default precision
# (see https://docs.python.org/3.7/library/decimal.html#decimal.DefaultContext):
ctx.prec = 28
# decimal with 29 digits
decimal = Decimal('1234567890123456789012345678.9')
assert decimal == loads(dumps(decimal))
assert decimal == loads(dumps(decimal, binary=False))
# negative decimal with 29 digits
decimal = Decimal('-1234567890123456789012345678.9')
assert decimal == loads(dumps(decimal))
assert decimal == loads(dumps(decimal, binary=False))
def test_int():
if is_pypy:
return
ion_value = loads(dumps(-123))
assert isinstance(ion_value, IonPyInt) and ion_value.ion_type == IonType.INT
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '-123'
def test_float_nan():
if is_pypy:
return
ion_value = loads(dumps(float("NaN")))
assert isinstance(ion_value, IonPyFloat) and ion_value.ion_type == IonType.FLOAT
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == 'null'
def test_decimal_exp_large_negative():
if is_pypy:
return
ion_value = loads(dumps(Decimal('123.456e-34')))
assert isinstance(ion_value, IonPyDecimal) and ion_value.ion_type == IonType.DECIMAL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '1.23456e-32'
def test_bool():
if is_pypy:
return
ion_value = loads(dumps(False))
assert isinstance(ion_value, IonPyBool) and ion_value.ion_type == IonType.BOOL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == 'false'
def test_dumps(p):
# test dumps
res = dumps(p.obj, sequence_as_stream=p.stream)
if not p.has_symbols:
assert (b'$ion_1_0 ' + p.expected).decode('utf-8') == res
else:
# The payload contains a LST. The value comes last, so compare the end bytes.
assert (p.expected).decode('utf-8') == res[len(res) - len(p.expected):]
def test_list():
if is_pypy:
return
ion_value = loads(dumps([six.text_type("Ion"), 123]))
assert isinstance(ion_value, IonPyList) and ion_value.ion_type == IonType.LIST
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '["Ion", 123]'
def test_symbol():
if is_pypy:
return
ion_value = loads(dumps(SymbolToken(six.text_type("Symbol"), None)))
assert isinstance(ion_value, IonPySymbol) and ion_value.ion_type == IonType.SYMBOL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"Symbol"'
def test_clob():
if is_pypy:
return
ion_value = loads(dumps(IonPyBytes.from_value(IonType.CLOB, bytearray.fromhex("06 49 6f 6e 06"))))
assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.CLOB
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"\\u0006Ion\\u0006"'
def test_string():
if is_pypy:
return
ion_value = loads(dumps(six.text_type("String")))
assert isinstance(ion_value, IonPyText) and ion_value.ion_type == IonType.STRING
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"String"'
def test_annotation_suppression():
if is_pypy:
return
ion_value = loads(dumps(IonPyInt.from_value(IonType.INT, 123, six.text_type("Annotation"))))
assert isinstance(ion_value, IonPyInt) and ion_value.ion_type == IonType.INT
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '123'
def test_pretty_print(p):
ion_text, indent, exact_text, regexes = p
ion_value = loads(ion_text)
actual_pretty_ion_text = dumps(ion_value, binary=False, indent=indent)
if exact_text is not None:
assert actual_pretty_ion_text == exact_text
for regex_str in regexes:
assert re.search(regex_str, actual_pretty_ion_text, re.M) is not None
def _test_name(ion_test):
if len(ion_test.ion_annotations) > 0:
test_name = ion_test.ion_annotations[0].text
else:
test_name = str(ion.dumps(ion_test['ion'], binary=False))
if test_name.startswith(_IVM):
test_name = test_name[len(_IVM):]
return " " + test_name
def print_result(cursor):
result_counter = 0
for row in cursor:
# Each row would be in Ion format.
logger.info(
dumps(row, binary=False, indent=' ', omit_version_marker=True))
result_counter += 1
return result_counter
def test_insert_multiple_documents(self):
# Given.
# Create Ion structs to insert.
parameter_1 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_1}))
parameter_2 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_2}))
query = "INSERT INTO {} <<?, ?>>".format(TABLE_NAME)
def execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2):
cursor = txn.execute_statement(query, parameter_1, parameter_2)
count = 0
for row in cursor:
count += 1
return count
# When.
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2))
self.assertEqual(2, count)
# Then.
search_query = "SELECT VALUE {} FROM {} WHERE {} IN (?,?)".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
ion_string_1 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_1))
ion_string_2 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_2))
def execute_statement_with_parameters_and_return_list_of_values(
txn, query, *parameters):
cursor = txn.execute_statement(query, *parameters)
values = list()
for row in cursor:
values.append(row)
return values
values = self.qldb_driver.execute_lambda(
lambda txn:
execute_statement_with_parameters_and_return_list_of_values(
txn, search_query, ion_string_1, ion_string_2))
self.assertTrue(MULTIPLE_DOCUMENT_VALUE_1 in values)
self.assertTrue(MULTIPLE_DOCUMENT_VALUE_2 in values)
def dump_func(*args, **kw):
sval = dumps(*args, **kw)
# encode to UTF-8 bytes for comparisons
return sval.encode('UTF-8')