def test_bool(): if is_pypy: return ion_value = loads(dumps(False)) assert isinstance(ion_value, IonPyBool) and ion_value.ion_type == IonType.BOOL json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == 'false'
def test_read_numerics_simpleion(): # http://amzn.github.io/ion-docs/guides/cookbook.html#reading-numeric-types data = u'1.23456 1.2345e6 123456 12345678901234567890' values = simpleion.loads(data, single_value=False) assert isinstance(values[0], Decimal) assert isinstance(values[1], float) assert isinstance(values[2], int) assert isinstance(values[3], int)
def test_clob(): if is_pypy: return ion_value = loads(dumps(IonPyBytes.from_value(IonType.CLOB, bytearray.fromhex("06 49 6f 6e 06")))) assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.CLOB json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '"\\u0006Ion\\u0006"'
def test_blob(): if is_pypy: return ion_value = loads(dumps(b64encode("Ion".encode("ASCII")) if six.PY2 else bytes("Ion", "ASCII"))) assert isinstance(ion_value, IonPyBytes) and ion_value.ion_type == IonType.BLOB json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '"SW9u"'
def test_symbol(): if is_pypy: return ion_value = loads(dumps(SymbolToken(six.text_type("Symbol"), None))) assert isinstance(ion_value, IonPySymbol) and ion_value.ion_type == IonType.SYMBOL json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '"Symbol"'
def test_string(): if is_pypy: return ion_value = loads(dumps(six.text_type("String"))) assert isinstance(ion_value, IonPyText) and ion_value.ion_type == IonType.STRING json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '"String"'
def test_decimal_exp_large_negative(): if is_pypy: return ion_value = loads(dumps(Decimal('123.456e-34'))) assert isinstance(ion_value, IonPyDecimal) and ion_value.ion_type == IonType.DECIMAL json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '1.23456e-32'
def test_timestamp(): if is_pypy: return ion_value = loads(dumps(datetime.datetime(2010, 6, 15, 3, 30, 45))) assert isinstance(ion_value, IonPyTimestamp) and ion_value.ion_type == IonType.TIMESTAMP json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '"2010-06-15 03:30:45"'
def test_float_inf(): if is_pypy: return ion_value = loads(dumps(float("Inf"))) assert isinstance(ion_value, IonPyFloat) and ion_value.ion_type == IonType.FLOAT json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == 'null'
def test_pretty_print(p): ion_text, indent, exact_text, regexes = p ion_value = loads(ion_text) actual_pretty_ion_text = dumps(ion_value, binary=False, indent=indent) if exact_text is not None: assert actual_pretty_ion_text == exact_text for regex_str in regexes: assert re.search(regex_str, actual_pretty_ion_text, re.M) is not None
def test_writing_simpleion_dump(): # http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data data = u'{hello: "world"}' value = simpleion.loads(data) ion = BytesIO() simpleion.dump(value, ion, binary=True) assert b'\xe0\x01\x00\xea\xec\x81\x83\xde\x88\x87\xb6\x85hello\xde\x87\x8a\x85world' == ion.getvalue( )
def test_list(): if is_pypy: return ion_value = loads(dumps([six.text_type("Ion"), 123])) assert isinstance(ion_value, IonPyList) and ion_value.ion_type == IonType.LIST json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '["Ion", 123]'
def test_annotation_suppression(): if is_pypy: return ion_value = loads(dumps(IonPyInt.from_value(IonType.INT, 123, six.text_type("Annotation")))) assert isinstance(ion_value, IonPyInt) and ion_value.ion_type == IonType.INT json_string = json.dumps(ion_value, cls=IonToJSONEncoder) assert json_string == '123'
def test_read_with_shared_symbol_table_simpleion(): # http://amzn.github.io/ion-docs/guides/cookbook.html#using-a-shared-symbol-table data = write_with_shared_symbol_table_simpleion() table = shared_symbol_table(u'test.csv.columns', 1, (u'id', u'type', u'state')) catalog = SymbolTableCatalog() catalog.register(table) values = simpleion.loads(data, catalog=catalog, single_value=False) assert values[2][u'id'] == 3
def test_quote_symbols(p): symbol_text, needs_quotes, backslash_required = p try: loads(symbol_text + '::4') threw_without_quotes = False except IonException as e: threw_without_quotes = True quoted_symbol_text = "'" + ('\\' if backslash_required else '') + symbol_text + "'" ion_value = loads(quoted_symbol_text + "::4") quoted = "'" in dumps(ion_value, binary=False) assert needs_quotes == threw_without_quotes assert needs_quotes == quoted assert len(ion_value.ion_annotations) == 1 assert ion_value.ion_annotations[0].text == symbol_text
def test_sparse_reads_simpleion(): # http://amzn.github.io/ion-docs/guides/cookbook.html#performing-sparse-reads data = sparse_reads_data() # The binary Ion equivalent of the above data. values = simpleion.loads(data, single_value=False) sum = 0 for value in values: if u'foo' == value.ion_annotations[0].text: sum += value[u'quantity'] assert 20 == sum
def test_insert_multiple_documents(self): # Given. # Create Ion structs to insert. parameter_1 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_1})) parameter_2 = loads(dumps({COLUMN_NAME: MULTIPLE_DOCUMENT_VALUE_2})) query = "INSERT INTO {} <<?, ?>>".format(TABLE_NAME) def execute_statement_with_parameters_and_return_count( txn, query, parameter_1, parameter_2): cursor = txn.execute_statement(query, parameter_1, parameter_2) count = 0 for row in cursor: count += 1 return count # When. count = self.qldb_driver.execute_lambda( lambda txn: execute_statement_with_parameters_and_return_count( txn, query, parameter_1, parameter_2)) self.assertEqual(2, count) # Then. search_query = "SELECT VALUE {} FROM {} WHERE {} IN (?,?)".format( COLUMN_NAME, TABLE_NAME, COLUMN_NAME) ion_string_1 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_1)) ion_string_2 = loads(dumps(MULTIPLE_DOCUMENT_VALUE_2)) def execute_statement_with_parameters_and_return_list_of_values( txn, query, *parameters): cursor = txn.execute_statement(query, *parameters) values = list() for row in cursor: values.append(row) return values values = self.qldb_driver.execute_lambda( lambda txn: execute_statement_with_parameters_and_return_list_of_values( txn, search_query, ion_string_1, ion_string_2)) self.assertTrue(MULTIPLE_DOCUMENT_VALUE_1 in values) self.assertTrue(MULTIPLE_DOCUMENT_VALUE_2 in values)
def verify_registration(driver, ledger_name, vin): """ Verify each version of the registration for the given VIN. :type driver: :py:class:`pyqldb.driver.qldb_driver.QldbDriver` :param driver: An instance of the QldbDriver class. :type ledger_name: str :param ledger_name: The ledger to get digest from. :type vin: str :param vin: VIN to query the revision history of a specific registration with. :raises AssertionError: When verification failed. """ logger.info("Let's verify the registration with VIN = {}, in ledger = {}.".format(vin, ledger_name)) digest = get_digest_result(ledger_name) digest_bytes = digest.get('Digest') digest_tip_address = digest.get('DigestTipAddress') logger.info('Got a ledger digest: digest tip address = {}, digest = {}.'.format( value_holder_to_string(digest_tip_address.get('IonText')), to_base_64(digest_bytes))) logger.info('Querying the registration with VIN = {} to verify each version of the registration...'.format(vin)) cursor = lookup_registration_for_vin(driver, vin) logger.info('Getting a proof for the document.') for row in cursor: block_address = row.get('blockAddress') document_id = row.get('metadata').get('id') result = get_revision(ledger_name, document_id, block_address_to_dictionary(block_address), digest_tip_address) revision = result.get('Revision').get('IonText') document_hash = loads(revision).get('hash') proof = result.get('Proof') logger.info('Got back a proof: {}.'.format(proof)) verified = verify_document(document_hash, digest_bytes, proof) if not verified: raise AssertionError('Document revision is not verified.') else: logger.info('Success! The document is verified.') altered_document_hash = flip_random_bit(document_hash) logger.info("Flipping one bit in the document's hash and assert that the document is NOT verified. " "The altered document hash is: {}.".format(to_base_64(altered_document_hash))) verified = verify_document(altered_document_hash, digest_bytes, proof) if verified: raise AssertionError('Expected altered document hash to not be verified against digest.') else: logger.info('Success! As expected flipping a bit in the document hash causes verification to fail.') logger.info('Finished verifying the registration with VIN = {} in ledger = {}.'.format(vin, ledger_name))
def execute_statement(session_token: str, transaction_id: str, statement: str): response = client.send_command(SessionToken=session_token, ExecuteStatement={ 'TransactionId': transaction_id, 'Statement': statement }) return [ ion.loads(value['IonBinary']) for value in response['ExecuteStatement']['FirstPage']['Values'] ]
def convert_object_to_ion(py_object): """ Convert a Python object into an Ion object. :type py_object: object :param py_object: The object to convert. :rtype: :py:class:`amazon.ion.simple_types.IonPyValue` :return: The converted Ion object. """ ion_object = loads(dumps(py_object)) return ion_object
def _test_data(algorithm): path = abspath( join(abspath(__file__), '..', '..', 'ion-hash-test', 'ion_hash_tests.ion')) f = open(path) ion_tests = ion.loads(f.read(), single_value=False) f.close() def _has_algorithm(ion_test): return algorithm in ion_test['expect'] return filter(_has_algorithm, ion_tests)
def get_data_file_keys_from_manifest(manifest_object): """ Retrieve the ordered list of data object keys within the given final manifest. :type manifest_object: str :param manifest_object: The content of the final manifest. :rtype: list :return: List of data object keys. """ ion_keys = loads(manifest_object).get('keys') list_of_keys = list(ion_keys) return list_of_keys
def _to_ion(obj): """ Check if the object is of Ion type; if not, convert to Ion. :raises TypeError in case conversion fails. """ if not hasattr(obj, "ion_annotations"): try: obj = loads(dumps(obj)) except TypeError: raise TypeError("Failed to convert parameter to Ion; unsupported data type: %r" % (type(obj))) return obj
def lambda_handler(event, context): """ Triggered for a batch of kinesis records. Parses QLDB Journal streams and sends an SNS notification for Person and Vehicle Registration Events. """ sns_topic_arn = os.environ['SNS_ARN'] raw_kinesis_records = event['Records'] # Deaggregate all records in one call records = deaggregate_records(raw_kinesis_records) # Iterate through deaggregated records for record in records: # Kinesis data in Python Lambdas is base64 encoded payload = base64.b64decode(record['kinesis']['data']) # payload is the actual ion binary record published by QLDB to the stream ion_record = ion.loads(payload) print("Ion reocord: ", (ion.dumps(ion_record, binary=False))) if (("recordType" in ion_record) and (ion_record["recordType"] == REVISION_DETAILS_RECORD_TYPE)): revision_data, revision_metadata = get_data_metdata_from_revision_record( ion_record) table_info = get_table_info_from_revision_record(ion_record) if (revision_metadata["version"] == 0): # a new record inserted if (table_info and table_info["tableName"] == PERSON_TABLENAME and person_data_has_required_fields(revision_data)): send_sns_notification( sns_topic_arn, 'New User Registered. Name: {first_name} {last_name}'. format(first_name=revision_data["FirstName"], last_name=revision_data["LastName"])) elif (table_info and table_info["tableName"] == VEHICLE_REGISTRATION_TABLENAME and vehicle_registration_data_has_required_fields( revision_data)): send_sns_notification( sns_topic_arn, 'New Vehicle Registered. ' 'VIN: {vin}, LicensePlateNumber: {license_plate_number}' .format(vin=revision_data["VIN"], license_plate_number=revision_data[ "LicensePlateNumber"])) else: print("No Action") return {'statusCode': 200}
def handler(event, context): """ Creates document for ledger's table from data obtained from AWS SQS message. """ qldb_driver = QldbDriver(ledger_name='my-db', region_name='us-west-2') data = json.loads(event['Records'][0]['body']) data['datetime'] = datetime.now() document_to_insert = loads(dumps(data)) with qldb_driver as driver: driver.execute_lambda(lambda x: insert_document(x, document_to_insert)) return {'statusCode': 200, 'body': 'Lambda triggered!'}
def test_execute_lambda_that_does_not_return_value(self): # Given. # Insert Ion struct to insert. ion_struct = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE})) # When. query = "INSERT INTO {} ?".format(TABLE_NAME) self.qldb_driver.execute_lambda( lambda txn: txn.execute_statement(query, ion_struct)) # Then. search_query = "SELECT VALUE {} FROM {} WHERE {} = ?".format( COLUMN_NAME, TABLE_NAME, COLUMN_NAME) ion_string = loads(dumps(SINGLE_DOCUMENT_VALUE)) def execute_statement_and_return_value(txn, query, *parameters): cursor = txn.execute_statement(query, *parameters) return next(cursor) value = self.qldb_driver.execute_lambda( lambda txn: execute_statement_and_return_value( txn, search_query, ion_string)) self.assertEqual(SINGLE_DOCUMENT_VALUE, value)
def test_update_ion_types(self): # Given. # Create Ion struct to insert. ion_value = loads(dumps({COLUMN_NAME: SINGLE_DOCUMENT_VALUE})) def execute_statement_and_return_count(txn, query, *parameter): cursor = txn.execute_statement(query, *parameter) count = 0 for row in cursor: count += 1 return count # Insert first record that will be subsequently updated. query = "INSERT INTO {} ?".format(TABLE_NAME) count = self.qldb_driver.execute_lambda( lambda txn: execute_statement_and_return_count( txn, query, ion_value)) self.assertEqual(1, count) # Use subTest context manager to setup parameterized tests. for ion_value in create_ion_values(): with self.subTest(ion_value=ion_value): # When. query = "UPDATE {} SET {} = ?".format(TABLE_NAME, COLUMN_NAME) count = self.qldb_driver.execute_lambda( lambda txn: execute_statement_and_return_count( txn, query, ion_value)) self.assertEqual(1, count) def execute_statement_and_return_value(txn, query, *parameters): cursor = txn.execute_statement(query, *parameters) return next(cursor) # Then. if isinstance(ion_value, IonPyNull): search_query = "SELECT VALUE {} FROM {} WHERE {} IS NULL".format( COLUMN_NAME, TABLE_NAME, COLUMN_NAME) value = self.qldb_driver.execute_lambda( lambda txn: execute_statement_and_return_value( txn, search_query)) else: search_query = "SELECT VALUE {} FROM {} WHERE {} = ?".format( COLUMN_NAME, TABLE_NAME, COLUMN_NAME) value = self.qldb_driver.execute_lambda( lambda txn: execute_statement_and_return_value( txn, search_query, ion_value)) self.assertEqual(ion_value.ion_type, value.ion_type)
def parse_block(value_holder): """ Parse the Block object returned by QLDB and retrieve block hash. :type value_holder: dict :param value_holder: A structure containing an Ion string value. :rtype: :py:class:`amazon.ion.simple_types.IonPyBytes` :return: The block hash. """ value_holder = value_holder.get('IonText') block = loads(value_holder) block_hash = block.get('blockHash') return block_hash
def value_holder_to_string(value_holder): """ Returns the string representation of a given `value_holder`. :type value_holder: dict :param value_holder: The `value_holder` to convert to string. :rtype: str :return: The string representation of the supplied `value_holder`. """ ret_val = dumps(loads(value_holder), binary=False, indent=' ', omit_version_marker=True) val = '{{ IonText: {}}}'.format(ret_val) return val
def to_qldb_hash(value): """ The QldbHash of an IonValue is just the IonHash of that value. :type value: str/:py:class:`amazon.ion.simple_types.IonSymbol` :param value: The string or Ion value to be converted to Ion hash. :rtype: :py:class:`pyqldb.util.qldb_hash.Qldbhash`/object :return: An QldbHash object that contains Ion hash. """ if isinstance(value, str): value = loads(dumps(value)) ion_hash = value.ion_hash('SHA256') ion_hash_digest = ion_hash return QldbHash(ion_hash_digest)