def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results-933886674506-eu-west-1/'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
if test_name == "Test routine replace_placeholders_in_sql":
self.test_list_of_commands = [
'CREATE __client__ IF NOT EXISTS __bucket__',
'DROP DATABASE IF EXISTS __client_____bucket__'
]
self.test_dict_of_replacements = {
'__bucket__': 'unittest',
'__client__': "batman"
}
self.test_list_of_updated_commands = [
'CREATE batman IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS batman_unittest'
]
def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results/'
self.success_query = 'CREATE DATABASE IF NOT EXISTS unittest'
self.fail_query = 'DROP DATABASE unittest'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
class TestAthenaManager(unittest.TestCase):
def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results/'
self.success_query = 'CREATE DATABASE IF NOT EXISTS unittest'
self.fail_query = 'DROP DATABASE unittest'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
def test_exec_sql_on_athena_response_contains_correct_keys(self):
"""Test routine exec_sql_on_athena_response_contains_correct_keys"""
response = AthenaManager.exec_sql_on_athena(self.athena_manager,
self.success_query,
self.test_ouput_location)
self.assertTrue(
'Status' in response and 'QueryExecutionId' in response
and 'Query' in response,
'Response does not include all required keys')
def test_on_success_exec_sql_on_athena_retruns_success_status(self):
"""Test routine on_success_exec_sql_on_athena_retruns_success_status"""
response = AthenaManager.exec_sql_on_athena(self.athena_manager,
self.success_query,
self.test_ouput_location)
self.assertTrue(response['Status'] == 'SUCCEEDED',
'SQL command was not successfull')
def test_on_failure_exec_sql_on_athena_raises_exception(self):
"""Test routine on_failure_exec_sql_on_athena_raises_exception"""
self.assertRaises(
Exception,
AthenaManager.exec_sql_on_athena(self.athena_manager,
self.fail_query,
self.test_ouput_location))
def lambda_handler(event, context):
'''
The entry point for the Lambda function
:param event: A dictionary containing all the information about the event that triggered the Lambda function
:param context: This object contains information about the state of the Lambda function itself.
:return: N/A
'''
# set environment variables
environment_name = event['ResourceProperties']['EnvnameProperty']
micro_environment = event['ResourceProperties']['MicroenvProperty']
client = event['ResourceProperties']['ClientProperty']
account = event['ResourceProperties']['AccountProperty']
ddl_bucket = event['ResourceProperties']['DDLBucketProperty']
ddl_create_key = event['ResourceProperties']['CreateSqlS3KeyProperty']
ddl_delete_key = event['ResourceProperties']['DeleteSqlS3KeyProperty']
data_bucket = event['ResourceProperties']['DataBucketProperty']
test_output_location = event['ResourceProperties'][
'AthenaSqlOutputUriProperty']
physical_resource_id = context.log_stream_name
# Create a dictionary of placeholders and their values that will be replaced in sql commands.
dict_of_replacements = {
"__bucket__": data_bucket,
"__envname__": environment_name,
"__microenv__": micro_environment,
"__client__": client,
"__account__": account
}
###Set AWS specific variables###
# Instantiate Connection
conn = Connection()
# Create an Athena connection
conn_athena = conn.athena_connection()
# Create an S3 connection
conn_s3 = conn.s3_connection()
# Instantiate S3Manager
s3_manager = S3Manager(conn_s3)
# Instantiate Athena Manager
athena_manager = AthenaManager(conn_athena)
try:
if event['RequestType'] != 'Delete':
# Download file that has the SQL Create Commands from s3
ddl_filename = s3_manager.download_object(ddl_bucket,
ddl_create_key)
logger.info('Downloaded {}'.format(ddl_filename))
response = execute_scripts(athena_manager, ddl_filename,
test_output_location,
dict_of_replacements, 'Create')
send_response(event, context, SUCCESS, response,
physical_resource_id)
return response
elif event['RequestType'] == 'Delete':
# Download file that has the SQL Delete Commands from s3
ddl_filename = s3_manager.download_object(ddl_bucket,
ddl_delete_key)
logger.info('Downloaded {}'.format(ddl_filename))
response = execute_scripts(athena_manager, ddl_filename,
test_output_location,
dict_of_replacements, 'Delete')
send_response(event, context, SUCCESS, response,
physical_resource_id)
return response
except Exception as error:
logger.info(error.args)
response_data = {'Reason': error.args}
send_response(event, context, FAILED, response_data,
physical_resource_id)
return error.args
def lambda_handler(event, context):
"""
This is the main entry point for lambda function
:param event: Event object that contains the bucket name and key of the manifest file that triggered the lambda
:param context: Contains context information about the lambda function itself e.g. remaining time
:return: Does not return anything
"""
# set environment
exec_environment = os.environ['exec_environment']
etl_file_path = os.environ['etl_file_path']
manifest_file_path = os.environ['manifest_file_path']
manifest_file = os.environ['manifest_file']
log_uri = os.environ['log_uri']
# Download manifest file
get_manifest_file(event, manifest_file_path, manifest_file)
# Instantiate Connection
conn = Connection()
# Create an EMR connection
conn_emr = conn.emr_connection()
# Create an S3 connection
conn_s3 = conn.s3_connection()
# Instantiate S3Manager
s3_manager = S3Manager()
# Parse the manifest file and generate etl from ETL template wth placeholder values filled in.
# Also gets the details about the EMR cluster to create.
try:
# Instantiate ManifestParser
manifest_parser = ManifestParser()
logger.info(
"Generating new ETL file from ETL template wth placeholder values filled in"
)
dest_etl_file = manifest_parser.parse_manifest_file(
manifest_file_path, manifest_file, conn_s3, s3_manager,
exec_environment)
logger.info("Generated: {}".format(dest_etl_file))
except:
logger.error(
'Failed while trying to generate a new ETL from s3://{}/{}'.format(
manifest_parser.script_s3_bucket,
manifest_parser.script_s3_key))
logging.error(sys.exc_info())
raise
# Copy generated ETL to S3. This will then be submitted to EMR
try:
s3_manager.upload_object(conn_s3, etl_file_path, dest_etl_file,
manifest_parser.script_s3_bucket,
'generated-etls', dest_etl_file)
except:
raise
# Launch and submit jobs to EMR
try:
# Instantiate EMR Instance
emr = EMRInstance()
cluster_name = "{}_{}".format(exec_environment,
manifest_parser.script_s3_key)
cluster_id = emr.get_first_available_cluster(conn_emr)
if manifest_parser.use_existing_cluster and cluster_id:
instance_groups = emr.get_instance_groups(conn_emr, cluster_id)
group_id = instance_groups['CORE']
instance_groups_count = emr.get_instance_groups_count(
conn_emr, cluster_id)
current_instance_count = instance_groups_count[group_id]
if manifest_parser.instance_count > current_instance_count:
emr.set_instance_count(conn_emr, cluster_id, group_id,
manifest_parser.instance_count)
# Allow 10 secs for resizing to start
time.sleep(10)
#submit job
emr.submit_job(
conn_emr, cluster_id, 's3://{}/generated-etls/{}'.format(
manifest_parser.script_s3_bucket, dest_etl_file),
dest_etl_file, 'cluster', 'CONTINUE')
else:
# Launch EMR cluster
emr.launch_emr_and_submit_job(
conn_emr, log_uri, 's3://{}/generated-etls/{}'.format(
manifest_parser.script_s3_bucket,
dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE',
'{}'.format(cluster_name), manifest_parser.terminate_cluster,
manifest_parser.instance_type, manifest_parser.instance_count)
logger.info("Submitted s3://{}/{} to process_{}".format(
manifest_parser.script, dest_etl_file, cluster_name))
except:
logger.error(
"Failed while trying to launch EMR cluster. Details below:")
raise
#!/usr/bin/Python
# IMPORT classes from aws package
from aws import Connection
from aws import EC2Instance
from aws import Volumes
# INSATANTIATE Connection
connInst = Connection()
# CREATE an EC2 Connection
reg = connInst.list_regions()
print reg
for regionaws in reg:
#rgname = reg
#print rgname
# return rgname
conn = connInst.ec2Connection(regionaws)
# PRINT connection
print conn
# Instantiate EC2Instance for operations
# ec2 = EC2Instance()
# InsID = ec2.list_instances(conn)
# print InsID
# for Ins in InsID:
# Ins.start()
# print (Ins, "is starting now")
# ec2 = EC2Instance()
#!/usr/bin/Python #Import classes from aws package from aws import Connection from aws import EC2Instance from aws import Volumes from aws import Security connInst = Connection() conn = connInst.ec2Connection() ''' Uncomment code as required. Replace instance_id with actual values ''' #Manage EC2 Instances ''' instance = EC2Instance() instance.list_instances(conn) instance.start_instance(conn, instance_id) instance.stop_instance(conn, instance_id) ''' ''' Uncomment code as required. Replace instance_id and volume_id with actual values ''' #Manage EBS Volumes ''' volumeInst = Volumes() volumeInst.list_volumes(conn) volumeInst.detach_volume(conn,volume_id) volumeInst.attach_volume(conn, volume_id, instance_id)
class TestSqlDeployCustomResourceLambda(unittest.TestCase):
def setUp(self):
"""Setup"""
test_name = self.shortDescription()
# Instantiate Connection
self.conn = Connection()
# Create an Athena connection
self.conn_athena = self.conn.athena_connection()
# Instantiate Athena Manager
self.athena_manager = AthenaManager(self.conn_athena)
self.test_ouput_location = 's3://aws-athena-query-results-933886674506-eu-west-1/'
if test_name == "Test routine get_sql_commands_from_file":
self.test_list_of_commands = [
'CREATE DATABASE IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS unittest'
]
if test_name == "Test routine replace_placeholders_in_sql":
self.test_list_of_commands = [
'CREATE __client__ IF NOT EXISTS __bucket__',
'DROP DATABASE IF EXISTS __client_____bucket__'
]
self.test_dict_of_replacements = {
'__bucket__': 'unittest',
'__client__': "batman"
}
self.test_list_of_updated_commands = [
'CREATE batman IF NOT EXISTS unittest',
'DROP DATABASE IF EXISTS batman_unittest'
]
def test_get_sql_commands_from_file(self):
"""Test routine get_sql_commands_from_file"""
list_of_commands = s.get_sql_commands_from_file('test_split.txt', ';')
self.assertEqual(self.test_list_of_commands, list_of_commands,
"Lists of commands don't match")
def test_execute_scripts_with_create_event(self):
"""Test routine execute_scripts"""
dict_of_responses = s.execute_scripts(self.athena_manager,
'test_split.txt',
self.test_ouput_location,
'Create')
self.assertTrue(
dict_of_responses['Create'][0]['Status'] == 'SUCCEEDED',
'First command failed.')
self.assertTrue(
dict_of_responses['Create'][1]['Status'] == 'SUCCEEDED',
'Second command failed.')
def test_execute_scripts_with_delete_event(self):
"""Test routine execute_scripts"""
dict_of_responses = s.execute_scripts(self.athena_manager,
'test_split.txt',
self.test_ouput_location,
'Delete')
self.assertTrue(
dict_of_responses['Delete'][0]['Status'] == 'SUCCEEDED',
'First command failed.')
self.assertTrue(
dict_of_responses['Delete'][1]['Status'] == 'SUCCEEDED',
'Second command failed.')
def test_replace_placeholders_in_sql(self):
"""Test routine replace_placeholders_in_sql"""
list_of_updated_commands = s.replace_placeholders_in_sql(
self.test_list_of_commands, self.test_dict_of_replacements)
self.assertEqual(list_of_updated_commands,
self.test_list_of_updated_commands,
"The placeholder replacement failed.")