class TestAthenaManager(unittest.TestCase):
def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results/'
self.success_query = 'CREATE DATABASE IF NOT EXISTS unittest'
self.fail_query = 'DROP DATABASE unittest'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
def test_exec_sql_on_athena_response_contains_correct_keys(self):
"""Test routine exec_sql_on_athena_response_contains_correct_keys"""
response = AthenaManager.exec_sql_on_athena(self.athena_manager,
self.success_query,
self.test_ouput_location)
self.assertTrue(
'Status' in response and 'QueryExecutionId' in response
and 'Query' in response,
'Response does not include all required keys')
def test_on_success_exec_sql_on_athena_retruns_success_status(self):
"""Test routine on_success_exec_sql_on_athena_retruns_success_status"""
response = AthenaManager.exec_sql_on_athena(self.athena_manager,
self.success_query,
self.test_ouput_location)
self.assertTrue(response['Status'] == 'SUCCEEDED',
'SQL command was not successfull')
def test_on_failure_exec_sql_on_athena_raises_exception(self):
"""Test routine on_failure_exec_sql_on_athena_raises_exception"""
self.assertRaises(
Exception,
AthenaManager.exec_sql_on_athena(self.athena_manager,
self.fail_query,
self.test_ouput_location))
def lambda_handler(event, context):
'''
The entry point for the Lambda function
:param event: A dictionary containing all the information about the event that triggered the Lambda function
:param context: This object contains information about the state of the Lambda function itself.
:return: N/A
'''
# set environment variables
environment_name = event['ResourceProperties']['EnvnameProperty']
micro_environment = event['ResourceProperties']['MicroenvProperty']
client = event['ResourceProperties']['ClientProperty']
account = event['ResourceProperties']['AccountProperty']
ddl_bucket = event['ResourceProperties']['DDLBucketProperty']
ddl_create_key = event['ResourceProperties']['CreateSqlS3KeyProperty']
ddl_delete_key = event['ResourceProperties']['DeleteSqlS3KeyProperty']
data_bucket = event['ResourceProperties']['DataBucketProperty']
test_output_location = event['ResourceProperties'][
'AthenaSqlOutputUriProperty']
physical_resource_id = context.log_stream_name
# Create a dictionary of placeholders and their values that will be replaced in sql commands.
dict_of_replacements = {
"__bucket__": data_bucket,
"__envname__": environment_name,
"__microenv__": micro_environment,
"__client__": client,
"__account__": account
}
###Set AWS specific variables###
# Instantiate Connection
conn = Connection()
# Create an Athena connection
conn_athena = conn.athena_connection()
# Create an S3 connection
conn_s3 = conn.s3_connection()
# Instantiate S3Manager
s3_manager = S3Manager(conn_s3)
# Instantiate Athena Manager
athena_manager = AthenaManager(conn_athena)
try:
if event['RequestType'] != 'Delete':
# Download file that has the SQL Create Commands from s3
ddl_filename = s3_manager.download_object(ddl_bucket,
ddl_create_key)
logger.info('Downloaded {}'.format(ddl_filename))
response = execute_scripts(athena_manager, ddl_filename,
test_output_location,
dict_of_replacements, 'Create')
send_response(event, context, SUCCESS, response,
physical_resource_id)
return response
elif event['RequestType'] == 'Delete':
# Download file that has the SQL Delete Commands from s3
ddl_filename = s3_manager.download_object(ddl_bucket,
ddl_delete_key)
logger.info('Downloaded {}'.format(ddl_filename))
response = execute_scripts(athena_manager, ddl_filename,
test_output_location,
dict_of_replacements, 'Delete')
send_response(event, context, SUCCESS, response,
physical_resource_id)
return response
except Exception as error:
logger.info(error.args)
response_data = {'Reason': error.args}
send_response(event, context, FAILED, response_data,
physical_resource_id)
return error.args
class TestSqlDeployCustomResourceLambda(unittest.TestCase):
def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results-933886674506-eu-west-1/'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
if test_name == "Test routine replace_placeholders_in_sql":
self.test_list_of_commands = [
'CREATE __client__ IF NOT EXISTS __bucket__',
'DROP DATABASE IF EXISTS __client_____bucket__'
]
self.test_dict_of_replacements = {
'__bucket__': 'unittest',
'__client__': "batman"
}
self.test_list_of_updated_commands = [
'CREATE batman IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS batman_unittest'
]
def test_get_sql_commands_from_file(self):
"""Test routine get_sql_commands_from_file"""
list_of_commands = s.get_sql_commands_from_file('test_split.txt', ';')
self.assertEqual(self.test_list_of_commands, list_of_commands,
"Lists of commands don't match")
def test_execute_scripts_with_create_event(self):
"""Test routine execute_scripts"""
dict_of_responses = s.execute_scripts(self.athena_manager,
'test_split.txt',
self.test_ouput_location,
'Create')
self.assertTrue(
dict_of_responses['Create'][0]['Status'] == 'SUCCEEDED',
'First command failed.')
self.assertTrue(
dict_of_responses['Create'][1]['Status'] == 'SUCCEEDED',
'Second command failed.')
def test_execute_scripts_with_delete_event(self):
"""Test routine execute_scripts"""
dict_of_responses = s.execute_scripts(self.athena_manager,
'test_split.txt',
self.test_ouput_location,
'Delete')
self.assertTrue(
dict_of_responses['Delete'][0]['Status'] == 'SUCCEEDED',
'First command failed.')
self.assertTrue(
dict_of_responses['Delete'][1]['Status'] == 'SUCCEEDED',
'Second command failed.')
def test_replace_placeholders_in_sql(self):
"""Test routine replace_placeholders_in_sql"""
list_of_updated_commands = s.replace_placeholders_in_sql(
self.test_list_of_commands, self.test_dict_of_replacements)
self.assertEqual(list_of_updated_commands,
self.test_list_of_updated_commands,
"The placeholder replacement failed.")