def do_create_table_index(transaction_executor, table_name,
index_attribute):
logger.info("Creating index on '{}'...".format(index_attribute))
statement = 'CREATE INDEX on {} ({})'.format(table_name,
index_attribute)
cursor = transaction_executor.execute_statement(statement)
return len(list(cursor))
def migrate(self, direction='UP'):
if direction == 'DOWN':
self.down()
elif direction == 'UP':
self.up()
else:
logger.info('Direction is missing!')
def get_block_with_proof(ledger_name, block_address, digest_tip_address):
"""
Get the block of a ledger's journal. Also returns a proof of the block for verification.
:type ledger_name: str
:param ledger_name: Name of the ledger to operate on.
:type block_address: dict
:param block_address: The location of the block to request.
:type digest_tip_address: dict
:param digest_tip_address: The location of the digest tip.
:rtype: dict
:return: The response of the request.
"""
logger.info(
"Let's get the block for block address {}, digest tip address {}, for the ledger named {}."
.format(block_address, digest_tip_address, ledger_name))
result = qldb.client().get_block(Name=ledger_name,
BlockAddress=block_address,
DigestTipAddress=digest_tip_address)
logger.info('Success. GetBlock: {}.'.format(
block_response_to_string(result)))
return result
def scan_table(transaction_executor, table_name, where='1=1', limit=10, offset=0):
"""
Scan for all the documents in a table.
:type transaction_executor: :py:class:`pyqldb.execution.executor.Executor`
:param transaction_executor: An Executor object allowing for execution of statements within a transaction.
:type table_name: str
:param table_name: The name of the table to operate on.
:type where: str
:param where: where condition.
:type limit: int
:param limit: query result limit
:type offset: int
:param offset: query result offset.
:rtype: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor`
:return: Cursor on the result set of a statement query.
"""
logger.info('Scanning {}...'.format(table_name))
# query = 'SELECT * FROM {table_name} WHERE {where} LIMIT {limit} OFFSET {offset}'.format(
query = 'SELECT * FROM {table_name} WHERE {where}'.format(
table_name=table_name,
where=where,
# limit=limit,
# offset=offset
)
return transaction_executor.execute_statement(query)
def ledger_list():
if DdlServices.ledgers:
logger.info('Loaded from cash: {}.'.format(DdlServices.ledgers))
return DdlServices.ledgers
result = qldb.client().list_ledgers()
DdlServices.ledgers = result.get('Ledgers')
return DdlServices.ledgers
def wait_for_active(ledger_name):
while True:
result = describe_ledger(ledger_name=ledger_name)
if result.get('State') == config.ACTIVE_STATE:
logger.info('Success. Ledger is active and ready to use.')
return result
sleep(config.LEDGER_CREATION_POLL_PERIOD_SEC)
def create_table_indexes(self, index_attributes=[]):
"""Create index on tables in a particular ledger."""
logger.info('Creating index on all tables in a single transaction...')
with qldb.session(self.ledger_name) as session:
for index_attribute in index_attributes:
session.execute_lambda(
lambda x: DdlServices.do_create_table_index(x, self.hash_table_name, index_attribute),
lambda retry_attempt: logger.info('Retrying due to OCC conflict...'))
logger.info('Index created successfully.')
def create_ledger(self):
"""Create a ledger and wait for it to be active."""
if self.ledger_exist():
logger.info('Ledger already exist!')
return True
DdlServices.do_create_ledger(self.ledger_name)
wait_for_active(self.ledger_name)
DdlServices.reset_ledgers()
def up(self):
try:
self.create_ledger()
if self.table_exist():
logger.info('Table already exist!')
return True
self.create_table()
self.create_table_indexes(['service_id', 'document_id', 'document_hash', 'field_list', 'op_mode'])
except Exception as e:
logger.exception('Migration UP failed! {}.'.format(e))
def ledger_exist(self):
"""
Returns information about a ledger, including its state and when it was created.
"""
try:
logger.info("Let's describe ledger...{}".format(self.ledger_name))
ledger = qldb.client().describe_ledger(Name=self.ledger_name)
logger.info('Success. describe ledger...{}.'.format(ledger))
return ledger
except Exception as e:
logger.exception('Unable to list ledgers!')
return None
def create_export(ledger_name, start_time, end_time, s3_bucket_name, s3_prefix,
encryption_configuration, role_arn):
"""
Request QLDB to export the contents of the journal for the given time period and S3 configuration. Before calling
this function the S3 bucket should be created,
see :py:meth:`hash_chain.ledger.export_journal.create_s3_bucket_if_not_exists`
:type ledger_name: str
:param ledger_name: Name of the ledger.
:type start_time: :py:class:`datetime.datetime`
:param start_time: Time from when the journal contents should be exported.
:type end_time: :py:class:`datetime.datetime`
:param end_time: Time until which the journal contents should be exported.
:type s3_bucket_name: str
:param s3_bucket_name: S3 bucket to write the data to.
:type s3_prefix: str
:param s3_prefix: S3 prefix to be prefixed to the files written.
:type encryption_configuration: dict
:param encryption_configuration: Encryption configuration for S3.
:type role_arn: str
:param role_arn: The IAM role ARN to be used when exporting the journal.
:rtype: dict
:return: The result of the request.
"""
logger.info(
"Let's create a journal export for ledger with name: {}.".format(
ledger_name))
try:
result = qldb.client().export_journal_to_s3(
Name=ledger_name,
InclusiveStartTime=start_time,
ExclusiveEndTime=end_time,
S3ExportConfiguration={
'Bucket': s3_bucket_name,
'Prefix': s3_prefix,
'EncryptionConfiguration': encryption_configuration
},
RoleArn=role_arn)
logger.info("Requested QLDB to export contents of the journal.")
return result
except qldb.client().exceptions.InvalidParameterException as ipe:
logger.error(
"The eventually consistent behavior of the IAM service may cause this export to fail its first"
" attempts, please retry.")
raise ipe
def describe_ledger(ledger_name):
"""
Returns information about a ledger, including its state and when it was created.
"""
try:
logger.info("Let's describe ledger...{}".format(ledger_name))
ledger = qldb.client().describe_ledger(Name=ledger_name)
logger.info('Success. describe ledger...{}.'.format(ledger))
return ledger
except Exception as e:
logger.exception('Unable to list ledgers!')
return fail_response('Unable to list ledgers!. Please try again.',
HTTPStatus.UNPROCESSABLE_ENTITY)
def get_table_data(ledger_name, table_name, where='1=1', limit=10, offset=0):
""" Scan for all the documents in a table."""
try:
with qldb.session(ledger_name) as session:
# Scan all the tables and print their documents.
tables = session.list_tables()
for table in tables:
cursor = session.execute_lambda(
lambda executor: DmlServices.scan_table(executor, table_name, where, limit, offset),
retry_indicator=lambda retry_attempt: logger.info('Retrying due to OCC conflict...'))
logger.info('Scan successful!')
return parse_result(cursor)
except Exception as e:
logger.exception('Unable to scan tables. {}'.format(e))
def table_exist(self):
"""
Connect to a session for a given ledger using default settings.
"""
try:
qldb_session = qldb.session(self.ledger_name)
logger.info('Listing table names ')
tables = qldb_session.list_tables()
_tables = []
for table in tables:
_tables.append(table)
return self.hash_table_name in _tables
except Exception as e:
logger.exception('Unable to create session.')
return False
def get_digest_result(ledger_name=config.LEDGER_NAME):
"""
Get the digest of a ledger's journal.
:type name: str
:param name: Name of the ledger to operate on.
:rtype: dict
:return: The digest in a 256-bit hash value and a block address.
"""
logger.info("Let's get the current digest of the ledger named {}".format(
ledger_name))
result = qldb.client().get_digest(Name=ledger_name)
logger.info('Success. LedgerDigest: {}.'.format(
digest_response_to_string(result)))
return result
def get_ledger_list():
"""
List all ledgers.
:rtype: list
:return: List of ledgers.
"""
try:
logger.info("Let's list all the ledgers...")
ledgers = DdlServices.ledger_list()
logger.info('Success. List of ledgers: {}.'.format(ledgers))
return ledgers
except Exception as e:
logger.exception('Unable to list ledgers!')
return fail_response('Unable to list ledgers!. Please try again.',
HTTPStatus.UNPROCESSABLE_ENTITY)
def drop_table(ledger_name=None, table_name=None):
"""Create a Table"""
try:
with qldb.session(ledger_name) as session:
session.execute_lambda(
lambda x: DdlServices.do_drop_table(x, table_name),
lambda retry_attempt: logger.info(
'Retrying due to OCC conflict...'))
logger.info('Table dropped successfully.')
return success_response('Table dropped successfully.',
HTTPStatus.CREATED)
except Exception:
logger.exception('Error creating table.')
return fail_response(
'Unable to create the table. Please try again.',
HTTPStatus.UNPROCESSABLE_ENTITY)
def create_export_and_wait_for_completion(name,
bucket,
prefix,
encryption_config,
role_arn=None):
"""
Request QLDB to export the contents of the journal for the given time period and S3 configuration. Before calling
this function the S3 bucket should be created, see
:py:class:`hash_chain.ledger.export_journal.create_s3_bucket_if_not_exists`
:type name: str
:param name: Name of the ledger to create a journal export for.
:type bucket: str
:param bucket: S3 bucket to write the data to.
:type prefix: str
:param prefix: S3 prefix to be prefixed to the files being written.
:type encryption_config: dict
:param encryption_config: Encryption configuration for S3.
:type role_arn: str
:param role_arn: The IAM role ARN to be used when exporting the journal.
:rtype: dict
:return: The result of the request.
"""
if role_arn is None:
role_arn = create_export_role(EXPORT_ROLE_NAME,
encryption_config.get('KmsKeyArn'),
ROLE_POLICY_NAME, bucket)
try:
start_time = datetime.utcnow() - timedelta(
minutes=JOURNAL_EXPORT_TIME_WINDOW_MINUTES)
end_time = datetime.utcnow()
result = create_export(name, start_time, end_time, bucket, prefix,
encryption_config, role_arn)
wait_for_export_to_complete(config.LEDGER_NAME, result.get('ExportId'))
logger.info('JournalS3Export for exportId {} is completed.'.format(
result.get('ExportId')))
return result
except Exception as e:
logger.exception('Unable to create an export!')
raise e
def insert_documents(ledger_name, table_name, documents):
"""
Insert documents into a table in a QLDB ledger.
"""
try:
with qldb.session(ledger_name) as session:
# An INSERT statement creates the initial revision of a document with a version number of zero.
# QLDB also assigns a unique document identifier in GUID format as part of the metadata.
session.execute_lambda(
lambda executor: DmlServices.do_insert_documents(executor, table_name, documents),
lambda retry_attempt: logger.info('Retrying due to OCC conflict...'))
logger.info('Documents inserted successfully!')
return success_response('Ledger is active and ready to use.', HTTPStatus.CREATED)
except Exception as e:
logger.exception('Error inserting or updating documents.')
return fail_response('Error inserting or updating documents. error: {}. Please try again.'.format(e),
HTTPStatus.UNPROCESSABLE_ENTITY)
def list_tables(ledger_name):
"""
Connect to a session for a given ledger using default settings.
"""
try:
qldb_session = qldb.session(ledger_name)
logger.info('Listing table names ')
tables = qldb_session.list_tables()
_tables = []
for table in tables:
_tables.append(table)
return _tables
except Exception as e:
logger.exception('Unable to create session.')
return fail_response(
'Unable to connect ledgers!. Please try again.',
HTTPStatus.UNPROCESSABLE_ENTITY)
def query_revision_history(qldb_session, table_name, condition_str, condition_value):
"""
Query revision history for a particular vehicle for verification.
:type qldb_session: :py:class:`pyqldb.session.qldb_session.QldbSession`
:param qldb_session: An instance of the QldbSession class.
:type vin: str
:param vin: VIN to query the revision history of a specific registration with.
:rtype: :py:class:`pyqldb.cursor.buffered_cursor.BufferedCursor`
:return: Cursor on the result set of the statement query.
"""
logger.info("Querying the '{}' table for the condition: {}...".format(table_name, condition_str))
query = 'SELECT * FROM _ql_committed_{} WHERE {}'.format(table_name, condition_str)
parameters = convert_object_to_ion(condition_value)
cursor = qldb_session.execute_statement(query, parameters)
return cursor
def print_result(cursor):
"""
Pretty print the result set. Returns the number of documents in the result set.
:type cursor: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor`/
:py:class:`pyqldb.cursor.buffered_cursor.BufferedCursor`
:param cursor: An instance of the StreamCursor or BufferedCursor class.
:rtype: int
:return: Number of documents in the result set.
"""
result_counter = 0
for row in cursor:
# Each row would be in Ion format.
logger.info(
dumps(row, binary=False, indent=' ', omit_version_marker=True))
result_counter += 1
return result_counter
def set_deletion_protection(ledger_name, deletion_protection):
"""
Update an existing ledger's deletion protection.
:type ledger_name: str
:param ledger_name: Name of the ledger to update.
:type deletion_protection: bool
:param deletion_protection: Enable or disable the deletion protection.
:rtype: dict
:return: Result from the request.
"""
logger.info(
"Let's set deletion protection to {} for the ledger with name {}.".
format(deletion_protection, ledger_name))
result = qldb.client().update_ledger(
Name=ledger_name, DeletionProtection=deletion_protection)
logger.info('Success. Ledger updated: {}'.format(result))
def create_table_index(ledger_name=None,
table_name=None,
index_attribute=None):
"""Create index on tables in a particular ledger."""
logger.info('Creating index on all tables in a single transaction...')
try:
with qldb.session(ledger_name) as session:
session.execute_lambda(
lambda x: DdlServices.do_create_table_index(
x, table_name, index_attribute), lambda retry_attempt:
logger.info('Retrying due to OCC conflict...'))
logger.info('Index created successfully.')
return success_response('Index created successfully.',
HTTPStatus.CREATED)
except Exception:
logger.exception('Unable to create index.')
return fail_response(
'Unable to create the index. Please try again.',
HTTPStatus.UNPROCESSABLE_ENTITY)
def create_export_role(role_name, key_arn, role_policy_name, s3_bucket):
"""
Create a new export rule and a new managed policy for the current AWS account.
:type role_name: str
:param role_name: The name of the role to be created.
:type key_arn: str
:param key_arn: The optional KMS Key ARN used to configure the role policy statement.
:type role_policy_name: str
:param role_policy_name: Name of the role policy to be created.
:type s3_bucket: str
:param s3_bucket: If key_arn is None, create a new ARN using the given bucket name.
:rtype: str
:return: The ARN of the newly created export role.
"""
iam_client = client('iam')
logger.info('Trying to retrieve role with name: {}.'.format(role_name))
try:
new_role_arn = iam_client.get_role(
RoleName=role_name).get('Role').get('Arn')
logger.info('The role called {} already exists.'.format(role_name))
except iam_client.exceptions.NoSuchEntityException:
logger.info(
'The role called {} does not exist. Creating it now.'.format(
role_name))
role = iam_client.create_role(
RoleName=role_name, AssumeRolePolicyDocument=ASSUME_ROLE_POLICY)
new_role_arn = role.get('Role').get('Arn')
role_policy_statement = EXPORT_ROLE_S3_STATEMENT_TEMPLATE.replace(
'{bucket_name}', s3_bucket)
if key_arn is not None:
role_policy_statement = "{}, {}".format(
role_policy_statement,
EXPORT_ROLE_KMS_STATEMENT_TEMPLATE.replace(
'{kms_arn}', key_arn))
role_policy = POLICY_TEMPLATE.replace('{statements}',
role_policy_statement)
create_policy_result = iam_client.create_policy(
PolicyName=role_policy_name, PolicyDocument=role_policy)
iam_client.attach_role_policy(
RoleName=role_name,
PolicyArn=create_policy_result.get('Policy').get('Arn'))
logger.info('Role {} created with ARN: {} and policy: {}.'.format(
role_name, new_role_arn, role_policy))
return new_role_arn
def describe_journal_export(ledger_name, export_id):
"""
Describe a journal export.
:type ledger_name: str
:param ledger_name: The ledger from which the journal is being exported.
:type export_id: str
:param export_id: The ExportId of the journal.
:rtype: dict
:return: Result from the request.
"""
logger.info(
"Let's describe a journal export for ledger with name: {}, exportId: {}."
.format(ledger_name, export_id))
export_result = qldb.client().describe_journal_s3_export(
Name=ledger_name, ExportId=export_id)
logger.info('Export described. Result = {}.'.format(
export_result['ExportDescription']))
return export_result
def get_block(ledger_name, block_address):
"""
Get the block of a ledger's journal.
:type ledger_name: str
:param ledger_name: Name of the ledger to operate on.
:type block_address: dict
:param block_address: The location of the block to request.
:rtype: dict
:return: The response of the request.
"""
logger.info(
"Let's get the block for block address {} of the ledger named {}.".
format(block_address, ledger_name))
result = qldb.client().get_block(Name=ledger_name,
BlockAddress=block_address)
logger.info('Success. GetBlock: {}'.format(
block_response_to_string(result)))
return result
def wait_for_export_to_complete(name, export_id):
"""
Wait for the journal export to complete.
:type name: str
:param name: Name of the ledger to wait on.
:type export_id: str
:param export_id: The unique export ID of the journal export.
:rtype: dict
:return: A description of the journal export.
:raises RunTimeError: When the export fails to complete within a constant number of retries.
"""
logger.info(
'Waiting for JournalS3Export for {} to complete...'.format(export_id))
count = 0
while count < MAX_RETRY_COUNT:
export_description = describe_journal_export(
name, export_id).get('ExportDescription')
if export_description.get('Status') == "COMPLETED":
logger.info('JournalS3Export completed.')
return export_description
logger.info('JournalS3Export is still in progress. Please wait.')
sleep(EXPORT_COMPLETION_POLL_PERIOD_SEC)
count += 1
raise RuntimeError(
'Journal Export did not complete for {}.'.format(export_id))
def do_insert_documents(transaction_executor, table_name, documents):
"""
Insert the given list of documents into a table in a single transaction.
:type transaction_executor: :py:class:`pyqldb.execution.executor.Executor`
:param transaction_executor: An Executor object allowing for execution of statements within a transaction.
:type table_name: str
:param table_name: Name of the table to insert documents into.
:type documents: list
:param documents: List of documents to insert.
:rtype: list
:return: List of documents IDs for the newly inserted documents.
"""
logger.info('Inserting some documents in the {} table...'.format(table_name))
statement = 'INSERT INTO {} ?'.format(table_name)
cursor = transaction_executor.execute_statement(statement, convert_object_to_ion(documents))
list_of_document_ids = get_document_ids_from_dml_results(cursor)
return list_of_document_ids
def wait_for_deleted(ledger_name):
logger.info('Waiting for the ledger to be deleted...')
while True:
try:
describe_ledger(ledger_name)
logger.info('The ledger is still being deleted. Please wait...')
sleep(config.LEDGER_DELETION_POLL_PERIOD_SEC)
except qldb.client().exceptions.ResourceNotFoundException:
logger.info('Success. The ledger is deleted.')
break
