def test_bool():
if is_pypy:
return
ion_value = loads(dumps(False))
assert isinstance(ion_value, IonPyBool) and ion_value.ion_type == IonType.BOOL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == 'false'
def test_read_numerics_simpleion():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-numeric-types
data = u'1.23456 1.2345e6 123456 12345678901234567890'
values = simpleion.loads(data, single_value=False)
assert isinstance(values[0], Decimal)
assert isinstance(values[1], float)
assert isinstance(values[2], int)
assert isinstance(values[3], int)
def test_clob():
if is_pypy:
return
ion_value = loads(dumps(IonPyBytes.from_value(IonType.CLOB, bytearray.fromhex("06 49 6f 6e 06"))))
assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.CLOB
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"\\u0006Ion\\u0006"'
def test_blob():
if is_pypy:
return
ion_value = loads(dumps(b64encode("Ion".encode("ASCII")) if six.PY2 else bytes("Ion", "ASCII")))
assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.BLOB
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"SW9u"'
def test_symbol():
if is_pypy:
return
ion_value = loads(dumps(SymbolToken(six.text_type("Symbol"), None)))
assert isinstance(ion_value, IonPySymbol) and ion_value.ion_type == IonType.SYMBOL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"Symbol"'
def test_string():
if is_pypy:
return
ion_value = loads(dumps(six.text_type("String")))
assert isinstance(ion_value, IonPyText) and ion_value.ion_type == IonType.STRING
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"String"'
def test_decimal_exp_large_negative():
if is_pypy:
return
ion_value = loads(dumps(Decimal('123.456e-34')))
assert isinstance(ion_value, IonPyDecimal) and ion_value.ion_type == IonType.DECIMAL
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '1.23456e-32'
def test_timestamp():
if is_pypy:
return
ion_value = loads(dumps(datetime.datetime(2010, 6, 15, 3, 30, 45)))
assert isinstance(ion_value, IonPyTimestamp) and ion_value.ion_type == IonType.TIMESTAMP
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '"2010-06-15 03:30:45"'
def test_float_inf():
if is_pypy:
return
ion_value = loads(dumps(float("Inf")))
assert isinstance(ion_value, IonPyFloat) and ion_value.ion_type == IonType.FLOAT
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == 'null'
def test_pretty_print(p):
ion_text, indent, exact_text, regexes = p
ion_value = loads(ion_text)
actual_pretty_ion_text = dumps(ion_value, binary=False, indent=indent)
if exact_text is not None:
assert actual_pretty_ion_text == exact_text
for regex_str in regexes:
assert re.search(regex_str, actual_pretty_ion_text, re.M) is not None
def test_writing_simpleion_dump():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
data = u'{hello: "world"}'
value = simpleion.loads(data)
ion = BytesIO()
simpleion.dump(value, ion, binary=True)
assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == ion.getvalue(
)
def test_list():
if is_pypy:
return
ion_value = loads(dumps([six.text_type("Ion"), 123]))
assert isinstance(ion_value, IonPyList) and ion_value.ion_type == IonType.LIST
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '["Ion", 123]'
def test_annotation_suppression():
if is_pypy:
return
ion_value = loads(dumps(IonPyInt.from_value(IonType.INT, 123, six.text_type("Annotation"))))
assert isinstance(ion_value, IonPyInt) and ion_value.ion_type == IonType.INT
json_string = json.dumps(ion_value, cls=IonToJSONEncoder)
assert json_string == '123'
def test_read_with_shared_symbol_table_simpleion():
# http://amzn.github.io/ion-docs/guides/cookbook.html#using-a-shared-symbol-table
data = write_with_shared_symbol_table_simpleion()
table = shared_symbol_table(u'test.csv.columns', 1,
(u'id', u'type', u'state'))
catalog = SymbolTableCatalog()
catalog.register(table)
values = simpleion.loads(data, catalog=catalog, single_value=False)
assert values[2][u'id'] == 3
def test_quote_symbols(p):
symbol_text, needs_quotes, backslash_required = p
try:
loads(symbol_text + '::4')
threw_without_quotes = False
except IonException as e:
threw_without_quotes = True
quoted_symbol_text = "'" + ('\\' if backslash_required else
'') + symbol_text + "'"
ion_value = loads(quoted_symbol_text + "::4")
quoted = "'" in dumps(ion_value, binary=False)
assert needs_quotes == threw_without_quotes
assert needs_quotes == quoted
assert len(ion_value.ion_annotations) == 1
assert ion_value.ion_annotations[0].text == symbol_text
def test_sparse_reads_simpleion():
# http://amzn.github.io/ion-docs/guides/cookbook.html#performing-sparse-reads
data = sparse_reads_data() # The binary Ion equivalent of the above data.
values = simpleion.loads(data, single_value=False)
sum = 0
for value in values:
if u'foo' == value.ion_annotations[0].text:
sum += value[u'quantity']
assert 20 == sum
def test_insert_multiple_documents(self):
# Given.
# Create Ion structs to insert.
parameter_1 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_1}))
parameter_2 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_2}))
query = "INSERT INTO {} <<?, ?>>".format(TABLE_NAME)
def execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2):
cursor = txn.execute_statement(query, parameter_1, parameter_2)
count = 0
for row in cursor:
count += 1
return count
# When.
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_with_parameters_and_return_count(
txn, query, parameter_1, parameter_2))
self.assertEqual(2, count)
# Then.
search_query = "SELECT VALUE {} FROM {} WHERE {} IN (?,?)".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
ion_string_1 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_1))
ion_string_2 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_2))
def execute_statement_with_parameters_and_return_list_of_values(
txn, query, *parameters):
cursor = txn.execute_statement(query, *parameters)
values = list()
for row in cursor:
values.append(row)
return values
values = self.qldb_driver.execute_lambda(
lambda txn:
execute_statement_with_parameters_and_return_list_of_values(
txn, search_query, ion_string_1, ion_string_2))
self.assertTrue(MULTIPLE_DOCUMENT_VALUE_1 in values)
self.assertTrue(MULTIPLE_DOCUMENT_VALUE_2 in values)
def verify_registration(driver, ledger_name, vin):
"""
Verify each version of the registration for the given VIN.
:type driver: :py:class:`pyqldb.driver.qldb_driver.QldbDriver`
:param driver: An instance of the QldbDriver class.
:type ledger_name: str
:param ledger_name: The ledger to get digest from.
:type vin: str
:param vin: VIN to query the revision history of a specific registration with.
:raises AssertionError: When verification failed.
"""
logger.info("Let's verify the registration with VIN = {}, in ledger = {}.".format(vin, ledger_name))
digest = get_digest_result(ledger_name)
digest_bytes = digest.get('Digest')
digest_tip_address = digest.get('DigestTipAddress')
logger.info('Got a ledger digest: digest tip address = {}, digest = {}.'.format(
value_holder_to_string(digest_tip_address.get('IonText')), to_base_64(digest_bytes)))
logger.info('Querying the registration with VIN = {} to verify each version of the registration...'.format(vin))
cursor = lookup_registration_for_vin(driver, vin)
logger.info('Getting a proof for the document.')
for row in cursor:
block_address = row.get('blockAddress')
document_id = row.get('metadata').get('id')
result = get_revision(ledger_name, document_id, block_address_to_dictionary(block_address), digest_tip_address)
revision = result.get('Revision').get('IonText')
document_hash = loads(revision).get('hash')
proof = result.get('Proof')
logger.info('Got back a proof: {}.'.format(proof))
verified = verify_document(document_hash, digest_bytes, proof)
if not verified:
raise AssertionError('Document revision is not verified.')
else:
logger.info('Success! The document is verified.')
altered_document_hash = flip_random_bit(document_hash)
logger.info("Flipping one bit in the document's hash and assert that the document is NOT verified. "
"The altered document hash is: {}.".format(to_base_64(altered_document_hash)))
verified = verify_document(altered_document_hash, digest_bytes, proof)
if verified:
raise AssertionError('Expected altered document hash to not be verified against digest.')
else:
logger.info('Success! As expected flipping a bit in the document hash causes verification to fail.')
logger.info('Finished verifying the registration with VIN = {} in ledger = {}.'.format(vin, ledger_name))
def execute_statement(session_token: str, transaction_id: str, statement: str):
response = client.send_command(SessionToken=session_token,
ExecuteStatement={
'TransactionId': transaction_id,
'Statement': statement
})
return [
ion.loads(value['IonBinary'])
for value in response['ExecuteStatement']['FirstPage']['Values']
]
def convert_object_to_ion(py_object):
"""
Convert a Python object into an Ion object.
:type py_object: object
:param py_object: The object to convert.
:rtype: :py:class:`amazon.ion.simple_types.IonPyValue`
:return: The converted Ion object.
"""
ion_object = loads(dumps(py_object))
return ion_object
def _test_data(algorithm):
path = abspath(
join(abspath(__file__), '..', '..', 'ion-hash-test',
'ion_hash_tests.ion'))
f = open(path)
ion_tests = ion.loads(f.read(), single_value=False)
f.close()
def _has_algorithm(ion_test):
return algorithm in ion_test['expect']
return filter(_has_algorithm, ion_tests)
def get_data_file_keys_from_manifest(manifest_object):
"""
Retrieve the ordered list of data object keys within the given final manifest.
:type manifest_object: str
:param manifest_object: The content of the final manifest.
:rtype: list
:return: List of data object keys.
"""
ion_keys = loads(manifest_object).get('keys')
list_of_keys = list(ion_keys)
return list_of_keys
def _to_ion(obj):
"""
Check if the object is of Ion type; if not, convert to Ion.
:raises TypeError in case conversion fails.
"""
if not hasattr(obj, "ion_annotations"):
try:
obj = loads(dumps(obj))
except TypeError:
raise TypeError("Failed to convert parameter to Ion; unsupported data type: %r" % (type(obj)))
return obj
def lambda_handler(event, context):
"""
Triggered for a batch of kinesis records.
Parses QLDB Journal streams and sends an SNS notification for Person and Vehicle Registration Events.
"""
sns_topic_arn = os.environ['SNS_ARN']
raw_kinesis_records = event['Records']
# Deaggregate all records in one call
records = deaggregate_records(raw_kinesis_records)
# Iterate through deaggregated records
for record in records:
# Kinesis data in Python Lambdas is base64 encoded
payload = base64.b64decode(record['kinesis']['data'])
# payload is the actual ion binary record published by QLDB to the stream
ion_record = ion.loads(payload)
print("Ion reocord: ", (ion.dumps(ion_record, binary=False)))
if (("recordType" in ion_record) and
(ion_record["recordType"] == REVISION_DETAILS_RECORD_TYPE)):
revision_data, revision_metadata = get_data_metdata_from_revision_record(
ion_record)
table_info = get_table_info_from_revision_record(ion_record)
if (revision_metadata["version"] == 0): # a new record inserted
if (table_info and table_info["tableName"] == PERSON_TABLENAME
and person_data_has_required_fields(revision_data)):
send_sns_notification(
sns_topic_arn,
'New User Registered. Name: {first_name} {last_name}'.
format(first_name=revision_data["FirstName"],
last_name=revision_data["LastName"]))
elif (table_info and table_info["tableName"]
== VEHICLE_REGISTRATION_TABLENAME
and vehicle_registration_data_has_required_fields(
revision_data)):
send_sns_notification(
sns_topic_arn, 'New Vehicle Registered. '
'VIN: {vin}, LicensePlateNumber: {license_plate_number}'
.format(vin=revision_data["VIN"],
license_plate_number=revision_data[
"LicensePlateNumber"]))
else:
print("No Action")
return {'statusCode': 200}
def handler(event, context):
"""
Creates document for ledger's table from data obtained from AWS SQS message.
"""
qldb_driver = QldbDriver(ledger_name='my-db', region_name='us-west-2')
data = json.loads(event['Records'][0]['body'])
data['datetime'] = datetime.now()
document_to_insert = loads(dumps(data))
with qldb_driver as driver:
driver.execute_lambda(lambda x: insert_document(x, document_to_insert))
return {'statusCode': 200, 'body': 'Lambda triggered!'}
def test_execute_lambda_that_does_not_return_value(self):
# Given.
# Insert Ion struct to insert.
ion_struct = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE}))
# When.
query = "INSERT INTO {} ?".format(TABLE_NAME)
self.qldb_driver.execute_lambda(
lambda txn: txn.execute_statement(query, ion_struct))
# Then.
search_query = "SELECT VALUE {} FROM {} WHERE {} = ?".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
ion_string = loads(dumps(SINGLE_DOCUMENT_VALUE))
def execute_statement_and_return_value(txn, query, *parameters):
cursor = txn.execute_statement(query, *parameters)
return next(cursor)
value = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_value(
txn, search_query, ion_string))
self.assertEqual(SINGLE_DOCUMENT_VALUE, value)
def test_update_ion_types(self):
# Given.
# Create Ion struct to insert.
ion_value = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE}))
def execute_statement_and_return_count(txn, query, *parameter):
cursor = txn.execute_statement(query, *parameter)
count = 0
for row in cursor:
count += 1
return count
# Insert first record that will be subsequently updated.
query = "INSERT INTO {} ?".format(TABLE_NAME)
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_count(
txn, query, ion_value))
self.assertEqual(1, count)
# Use subTest context manager to setup parameterized tests.
for ion_value in create_ion_values():
with self.subTest(ion_value=ion_value):
# When.
query = "UPDATE {} SET {} = ?".format(TABLE_NAME, COLUMN_NAME)
count = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_count(
txn, query, ion_value))
self.assertEqual(1, count)
def execute_statement_and_return_value(txn, query,
*parameters):
cursor = txn.execute_statement(query, *parameters)
return next(cursor)
# Then.
if isinstance(ion_value, IonPyNull):
search_query = "SELECT VALUE {} FROM {} WHERE {} IS NULL".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
value = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_value(
txn, search_query))
else:
search_query = "SELECT VALUE {} FROM {} WHERE {} = ?".format(
COLUMN_NAME, TABLE_NAME, COLUMN_NAME)
value = self.qldb_driver.execute_lambda(
lambda txn: execute_statement_and_return_value(
txn, search_query, ion_value))
self.assertEqual(ion_value.ion_type, value.ion_type)
def parse_block(value_holder):
"""
Parse the Block object returned by QLDB and retrieve block hash.
:type value_holder: dict
:param value_holder: A structure containing an Ion string value.
:rtype: :py:class:`amazon.ion.simple_types.IonPyBytes`
:return: The block hash.
"""
value_holder = value_holder.get('IonText')
block = loads(value_holder)
block_hash = block.get('blockHash')
return block_hash
def value_holder_to_string(value_holder):
"""
Returns the string representation of a given `value_holder`.
:type value_holder: dict
:param value_holder: The `value_holder` to convert to string.
:rtype: str
:return: The string representation of the supplied `value_holder`.
"""
ret_val = dumps(loads(value_holder),
binary=False,
indent=' ',
omit_version_marker=True)
val = '{{ IonText: {}}}'.format(ret_val)
return val
def to_qldb_hash(value):
"""
The QldbHash of an IonValue is just the IonHash of that value.
:type value: str/:py:class:`amazon.ion.simple_types.IonSymbol`
:param value: The string or Ion value to be converted to Ion hash.
:rtype: :py:class:`pyqldb.util.qldb_hash.Qldbhash`/object
:return: An QldbHash object that contains Ion hash.
"""
if isinstance(value, str):
value = loads(dumps(value))
ion_hash = value.ion_hash('SHA256')
ion_hash_digest = ion_hash
return QldbHash(ion_hash_digest)
