def do_create_table_index(transaction_executor, table_name, index_attribute): logger.info("Creating index on '{}'...".format(index_attribute)) statement = 'CREATE INDEX on {} ({})'.format(table_name, index_attribute) cursor = transaction_executor.execute_statement(statement) return len(list(cursor))
def migrate(self, direction='UP'): if direction == 'DOWN': self.down() elif direction == 'UP': self.up() else: logger.info('Direction is missing!')
def get_block_with_proof(ledger_name, block_address, digest_tip_address): """ Get the block of a ledger's journal. Also returns a proof of the block for verification. :type ledger_name: str :param ledger_name: Name of the ledger to operate on. :type block_address: dict :param block_address: The location of the block to request. :type digest_tip_address: dict :param digest_tip_address: The location of the digest tip. :rtype: dict :return: The response of the request. """ logger.info( "Let's get the block for block address {}, digest tip address {}, for the ledger named {}." .format(block_address, digest_tip_address, ledger_name)) result = qldb.client().get_block(Name=ledger_name, BlockAddress=block_address, DigestTipAddress=digest_tip_address) logger.info('Success. GetBlock: {}.'.format( block_response_to_string(result))) return result
def scan_table(transaction_executor, table_name, where='1=1', limit=10, offset=0): """ Scan for all the documents in a table. :type transaction_executor: :py:class:`pyqldb.execution.executor.Executor` :param transaction_executor: An Executor object allowing for execution of statements within a transaction. :type table_name: str :param table_name: The name of the table to operate on. :type where: str :param where: where condition. :type limit: int :param limit: query result limit :type offset: int :param offset: query result offset. :rtype: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor` :return: Cursor on the result set of a statement query. """ logger.info('Scanning {}...'.format(table_name)) # query = 'SELECT * FROM {table_name} WHERE {where} LIMIT {limit} OFFSET {offset}'.format( query = 'SELECT * FROM {table_name} WHERE {where}'.format( table_name=table_name, where=where, # limit=limit, # offset=offset ) return transaction_executor.execute_statement(query)
def ledger_list(): if DdlServices.ledgers: logger.info('Loaded from cash: {}.'.format(DdlServices.ledgers)) return DdlServices.ledgers result = qldb.client().list_ledgers() DdlServices.ledgers = result.get('Ledgers') return DdlServices.ledgers
def wait_for_active(ledger_name): while True: result = describe_ledger(ledger_name=ledger_name) if result.get('State') == config.ACTIVE_STATE: logger.info('Success. Ledger is active and ready to use.') return result sleep(config.LEDGER_CREATION_POLL_PERIOD_SEC)
def create_table_indexes(self, index_attributes=[]): """Create index on tables in a particular ledger.""" logger.info('Creating index on all tables in a single transaction...') with qldb.session(self.ledger_name) as session: for index_attribute in index_attributes: session.execute_lambda( lambda x: DdlServices.do_create_table_index(x, self.hash_table_name, index_attribute), lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) logger.info('Index created successfully.')
def create_ledger(self): """Create a ledger and wait for it to be active.""" if self.ledger_exist(): logger.info('Ledger already exist!') return True DdlServices.do_create_ledger(self.ledger_name) wait_for_active(self.ledger_name) DdlServices.reset_ledgers()
def up(self): try: self.create_ledger() if self.table_exist(): logger.info('Table already exist!') return True self.create_table() self.create_table_indexes(['service_id', 'document_id', 'document_hash', 'field_list', 'op_mode']) except Exception as e: logger.exception('Migration UP failed! {}.'.format(e))
def ledger_exist(self): """ Returns information about a ledger, including its state and when it was created. """ try: logger.info("Let's describe ledger...{}".format(self.ledger_name)) ledger = qldb.client().describe_ledger(Name=self.ledger_name) logger.info('Success. describe ledger...{}.'.format(ledger)) return ledger except Exception as e: logger.exception('Unable to list ledgers!') return None
def create_export(ledger_name, start_time, end_time, s3_bucket_name, s3_prefix, encryption_configuration, role_arn): """ Request QLDB to export the contents of the journal for the given time period and S3 configuration. Before calling this function the S3 bucket should be created, see :py:meth:`hash_chain.ledger.export_journal.create_s3_bucket_if_not_exists` :type ledger_name: str :param ledger_name: Name of the ledger. :type start_time: :py:class:`datetime.datetime` :param start_time: Time from when the journal contents should be exported. :type end_time: :py:class:`datetime.datetime` :param end_time: Time until which the journal contents should be exported. :type s3_bucket_name: str :param s3_bucket_name: S3 bucket to write the data to. :type s3_prefix: str :param s3_prefix: S3 prefix to be prefixed to the files written. :type encryption_configuration: dict :param encryption_configuration: Encryption configuration for S3. :type role_arn: str :param role_arn: The IAM role ARN to be used when exporting the journal. :rtype: dict :return: The result of the request. """ logger.info( "Let's create a journal export for ledger with name: {}.".format( ledger_name)) try: result = qldb.client().export_journal_to_s3( Name=ledger_name, InclusiveStartTime=start_time, ExclusiveEndTime=end_time, S3ExportConfiguration={ 'Bucket': s3_bucket_name, 'Prefix': s3_prefix, 'EncryptionConfiguration': encryption_configuration }, RoleArn=role_arn) logger.info("Requested QLDB to export contents of the journal.") return result except qldb.client().exceptions.InvalidParameterException as ipe: logger.error( "The eventually consistent behavior of the IAM service may cause this export to fail its first" " attempts, please retry.") raise ipe
def describe_ledger(ledger_name): """ Returns information about a ledger, including its state and when it was created. """ try: logger.info("Let's describe ledger...{}".format(ledger_name)) ledger = qldb.client().describe_ledger(Name=ledger_name) logger.info('Success. describe ledger...{}.'.format(ledger)) return ledger except Exception as e: logger.exception('Unable to list ledgers!') return fail_response('Unable to list ledgers!. Please try again.', HTTPStatus.UNPROCESSABLE_ENTITY)
def get_table_data(ledger_name, table_name, where='1=1', limit=10, offset=0): """ Scan for all the documents in a table.""" try: with qldb.session(ledger_name) as session: # Scan all the tables and print their documents. tables = session.list_tables() for table in tables: cursor = session.execute_lambda( lambda executor: DmlServices.scan_table(executor, table_name, where, limit, offset), retry_indicator=lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) logger.info('Scan successful!') return parse_result(cursor) except Exception as e: logger.exception('Unable to scan tables. {}'.format(e))
def table_exist(self): """ Connect to a session for a given ledger using default settings. """ try: qldb_session = qldb.session(self.ledger_name) logger.info('Listing table names ') tables = qldb_session.list_tables() _tables = [] for table in tables: _tables.append(table) return self.hash_table_name in _tables except Exception as e: logger.exception('Unable to create session.') return False
def get_digest_result(ledger_name=config.LEDGER_NAME): """ Get the digest of a ledger's journal. :type name: str :param name: Name of the ledger to operate on. :rtype: dict :return: The digest in a 256-bit hash value and a block address. """ logger.info("Let's get the current digest of the ledger named {}".format( ledger_name)) result = qldb.client().get_digest(Name=ledger_name) logger.info('Success. LedgerDigest: {}.'.format( digest_response_to_string(result))) return result
def get_ledger_list(): """ List all ledgers. :rtype: list :return: List of ledgers. """ try: logger.info("Let's list all the ledgers...") ledgers = DdlServices.ledger_list() logger.info('Success. List of ledgers: {}.'.format(ledgers)) return ledgers except Exception as e: logger.exception('Unable to list ledgers!') return fail_response('Unable to list ledgers!. Please try again.', HTTPStatus.UNPROCESSABLE_ENTITY)
def drop_table(ledger_name=None, table_name=None): """Create a Table""" try: with qldb.session(ledger_name) as session: session.execute_lambda( lambda x: DdlServices.do_drop_table(x, table_name), lambda retry_attempt: logger.info( 'Retrying due to OCC conflict...')) logger.info('Table dropped successfully.') return success_response('Table dropped successfully.', HTTPStatus.CREATED) except Exception: logger.exception('Error creating table.') return fail_response( 'Unable to create the table. Please try again.', HTTPStatus.UNPROCESSABLE_ENTITY)
def create_export_and_wait_for_completion(name, bucket, prefix, encryption_config, role_arn=None): """ Request QLDB to export the contents of the journal for the given time period and S3 configuration. Before calling this function the S3 bucket should be created, see :py:class:`hash_chain.ledger.export_journal.create_s3_bucket_if_not_exists` :type name: str :param name: Name of the ledger to create a journal export for. :type bucket: str :param bucket: S3 bucket to write the data to. :type prefix: str :param prefix: S3 prefix to be prefixed to the files being written. :type encryption_config: dict :param encryption_config: Encryption configuration for S3. :type role_arn: str :param role_arn: The IAM role ARN to be used when exporting the journal. :rtype: dict :return: The result of the request. """ if role_arn is None: role_arn = create_export_role(EXPORT_ROLE_NAME, encryption_config.get('KmsKeyArn'), ROLE_POLICY_NAME, bucket) try: start_time = datetime.utcnow() - timedelta( minutes=JOURNAL_EXPORT_TIME_WINDOW_MINUTES) end_time = datetime.utcnow() result = create_export(name, start_time, end_time, bucket, prefix, encryption_config, role_arn) wait_for_export_to_complete(config.LEDGER_NAME, result.get('ExportId')) logger.info('JournalS3Export for exportId {} is completed.'.format( result.get('ExportId'))) return result except Exception as e: logger.exception('Unable to create an export!') raise e
def insert_documents(ledger_name, table_name, documents): """ Insert documents into a table in a QLDB ledger. """ try: with qldb.session(ledger_name) as session: # An INSERT statement creates the initial revision of a document with a version number of zero. # QLDB also assigns a unique document identifier in GUID format as part of the metadata. session.execute_lambda( lambda executor: DmlServices.do_insert_documents(executor, table_name, documents), lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) logger.info('Documents inserted successfully!') return success_response('Ledger is active and ready to use.', HTTPStatus.CREATED) except Exception as e: logger.exception('Error inserting or updating documents.') return fail_response('Error inserting or updating documents. error: {}. Please try again.'.format(e), HTTPStatus.UNPROCESSABLE_ENTITY)
def list_tables(ledger_name): """ Connect to a session for a given ledger using default settings. """ try: qldb_session = qldb.session(ledger_name) logger.info('Listing table names ') tables = qldb_session.list_tables() _tables = [] for table in tables: _tables.append(table) return _tables except Exception as e: logger.exception('Unable to create session.') return fail_response( 'Unable to connect ledgers!. Please try again.', HTTPStatus.UNPROCESSABLE_ENTITY)
def query_revision_history(qldb_session, table_name, condition_str, condition_value): """ Query revision history for a particular vehicle for verification. :type qldb_session: :py:class:`pyqldb.session.qldb_session.QldbSession` :param qldb_session: An instance of the QldbSession class. :type vin: str :param vin: VIN to query the revision history of a specific registration with. :rtype: :py:class:`pyqldb.cursor.buffered_cursor.BufferedCursor` :return: Cursor on the result set of the statement query. """ logger.info("Querying the '{}' table for the condition: {}...".format(table_name, condition_str)) query = 'SELECT * FROM _ql_committed_{} WHERE {}'.format(table_name, condition_str) parameters = convert_object_to_ion(condition_value) cursor = qldb_session.execute_statement(query, parameters) return cursor
def print_result(cursor): """ Pretty print the result set. Returns the number of documents in the result set. :type cursor: :py:class:`pyqldb.cursor.stream_cursor.StreamCursor`/ :py:class:`pyqldb.cursor.buffered_cursor.BufferedCursor` :param cursor: An instance of the StreamCursor or BufferedCursor class. :rtype: int :return: Number of documents in the result set. """ result_counter = 0 for row in cursor: # Each row would be in Ion format. logger.info( dumps(row, binary=False, indent=' ', omit_version_marker=True)) result_counter += 1 return result_counter
def set_deletion_protection(ledger_name, deletion_protection): """ Update an existing ledger's deletion protection. :type ledger_name: str :param ledger_name: Name of the ledger to update. :type deletion_protection: bool :param deletion_protection: Enable or disable the deletion protection. :rtype: dict :return: Result from the request. """ logger.info( "Let's set deletion protection to {} for the ledger with name {}.". format(deletion_protection, ledger_name)) result = qldb.client().update_ledger( Name=ledger_name, DeletionProtection=deletion_protection) logger.info('Success. Ledger updated: {}'.format(result))
def create_table_index(ledger_name=None, table_name=None, index_attribute=None): """Create index on tables in a particular ledger.""" logger.info('Creating index on all tables in a single transaction...') try: with qldb.session(ledger_name) as session: session.execute_lambda( lambda x: DdlServices.do_create_table_index( x, table_name, index_attribute), lambda retry_attempt: logger.info('Retrying due to OCC conflict...')) logger.info('Index created successfully.') return success_response('Index created successfully.', HTTPStatus.CREATED) except Exception: logger.exception('Unable to create index.') return fail_response( 'Unable to create the index. Please try again.', HTTPStatus.UNPROCESSABLE_ENTITY)
def create_export_role(role_name, key_arn, role_policy_name, s3_bucket): """ Create a new export rule and a new managed policy for the current AWS account. :type role_name: str :param role_name: The name of the role to be created. :type key_arn: str :param key_arn: The optional KMS Key ARN used to configure the role policy statement. :type role_policy_name: str :param role_policy_name: Name of the role policy to be created. :type s3_bucket: str :param s3_bucket: If key_arn is None, create a new ARN using the given bucket name. :rtype: str :return: The ARN of the newly created export role. """ iam_client = client('iam') logger.info('Trying to retrieve role with name: {}.'.format(role_name)) try: new_role_arn = iam_client.get_role( RoleName=role_name).get('Role').get('Arn') logger.info('The role called {} already exists.'.format(role_name)) except iam_client.exceptions.NoSuchEntityException: logger.info( 'The role called {} does not exist. Creating it now.'.format( role_name)) role = iam_client.create_role( RoleName=role_name, AssumeRolePolicyDocument=ASSUME_ROLE_POLICY) new_role_arn = role.get('Role').get('Arn') role_policy_statement = EXPORT_ROLE_S3_STATEMENT_TEMPLATE.replace( '{bucket_name}', s3_bucket) if key_arn is not None: role_policy_statement = "{}, {}".format( role_policy_statement, EXPORT_ROLE_KMS_STATEMENT_TEMPLATE.replace( '{kms_arn}', key_arn)) role_policy = POLICY_TEMPLATE.replace('{statements}', role_policy_statement) create_policy_result = iam_client.create_policy( PolicyName=role_policy_name, PolicyDocument=role_policy) iam_client.attach_role_policy( RoleName=role_name, PolicyArn=create_policy_result.get('Policy').get('Arn')) logger.info('Role {} created with ARN: {} and policy: {}.'.format( role_name, new_role_arn, role_policy)) return new_role_arn
def describe_journal_export(ledger_name, export_id): """ Describe a journal export. :type ledger_name: str :param ledger_name: The ledger from which the journal is being exported. :type export_id: str :param export_id: The ExportId of the journal. :rtype: dict :return: Result from the request. """ logger.info( "Let's describe a journal export for ledger with name: {}, exportId: {}." .format(ledger_name, export_id)) export_result = qldb.client().describe_journal_s3_export( Name=ledger_name, ExportId=export_id) logger.info('Export described. Result = {}.'.format( export_result['ExportDescription'])) return export_result
def get_block(ledger_name, block_address): """ Get the block of a ledger's journal. :type ledger_name: str :param ledger_name: Name of the ledger to operate on. :type block_address: dict :param block_address: The location of the block to request. :rtype: dict :return: The response of the request. """ logger.info( "Let's get the block for block address {} of the ledger named {}.". format(block_address, ledger_name)) result = qldb.client().get_block(Name=ledger_name, BlockAddress=block_address) logger.info('Success. GetBlock: {}'.format( block_response_to_string(result))) return result
def wait_for_export_to_complete(name, export_id): """ Wait for the journal export to complete. :type name: str :param name: Name of the ledger to wait on. :type export_id: str :param export_id: The unique export ID of the journal export. :rtype: dict :return: A description of the journal export. :raises RunTimeError: When the export fails to complete within a constant number of retries. """ logger.info( 'Waiting for JournalS3Export for {} to complete...'.format(export_id)) count = 0 while count < MAX_RETRY_COUNT: export_description = describe_journal_export( name, export_id).get('ExportDescription') if export_description.get('Status') == "COMPLETED": logger.info('JournalS3Export completed.') return export_description logger.info('JournalS3Export is still in progress. Please wait.') sleep(EXPORT_COMPLETION_POLL_PERIOD_SEC) count += 1 raise RuntimeError( 'Journal Export did not complete for {}.'.format(export_id))
def do_insert_documents(transaction_executor, table_name, documents): """ Insert the given list of documents into a table in a single transaction. :type transaction_executor: :py:class:`pyqldb.execution.executor.Executor` :param transaction_executor: An Executor object allowing for execution of statements within a transaction. :type table_name: str :param table_name: Name of the table to insert documents into. :type documents: list :param documents: List of documents to insert. :rtype: list :return: List of documents IDs for the newly inserted documents. """ logger.info('Inserting some documents in the {} table...'.format(table_name)) statement = 'INSERT INTO {} ?'.format(table_name) cursor = transaction_executor.execute_statement(statement, convert_object_to_ion(documents)) list_of_document_ids = get_document_ids_from_dml_results(cursor) return list_of_document_ids
def wait_for_deleted(ledger_name): logger.info('Waiting for the ledger to be deleted...') while True: try: describe_ledger(ledger_name) logger.info('The ledger is still being deleted. Please wait...') sleep(config.LEDGER_DELETION_POLL_PERIOD_SEC) except qldb.client().exceptions.ResourceNotFoundException: logger.info('Success. The ledger is deleted.') break