def setUp(self): """Setup""" test_name = self.shortDescription() # Instantiate Connection self.conn = Connection() # Create an Athena connection self.conn_athena = self.conn.athena_connection() # Instantiate Athena Manager self.athena_manager = AthenaManager(self.conn_athena) self.test_ouput_location = 's3://aws-athena-query-results-933886674506-eu-west-1/' if test_name == "Test routine get_sql_commands_from_file": self.test_list_of_commands = [ 'CREATE DATABASE IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS unittest' ] if test_name == "Test routine replace_placeholders_in_sql": self.test_list_of_commands = [ 'CREATE __client__ IF NOT EXISTS __bucket__', 'DROP DATABASE IF EXISTS __client_____bucket__' ] self.test_dict_of_replacements = { '__bucket__': 'unittest', '__client__': "batman" } self.test_list_of_updated_commands = [ 'CREATE batman IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS batman_unittest' ]
def setUp(self): """Setup""" test_name = self.shortDescription() # Instantiate Connection self.conn = Connection() # Create an Athena connection self.conn_athena = self.conn.athena_connection() # Instantiate Athena Manager self.athena_manager = AthenaManager(self.conn_athena) self.test_ouput_location = 's3://aws-athena-query-results/' self.success_query = 'CREATE DATABASE IF NOT EXISTS unittest' self.fail_query = 'DROP DATABASE unittest' if test_name == "Test routine get_sql_commands_from_file": self.test_list_of_commands = [ 'CREATE DATABASE IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS unittest' ]
class TestAthenaManager(unittest.TestCase): def setUp(self): """Setup""" test_name = self.shortDescription() # Instantiate Connection self.conn = Connection() # Create an Athena connection self.conn_athena = self.conn.athena_connection() # Instantiate Athena Manager self.athena_manager = AthenaManager(self.conn_athena) self.test_ouput_location = 's3://aws-athena-query-results/' self.success_query = 'CREATE DATABASE IF NOT EXISTS unittest' self.fail_query = 'DROP DATABASE unittest' if test_name == "Test routine get_sql_commands_from_file": self.test_list_of_commands = [ 'CREATE DATABASE IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS unittest' ] def test_exec_sql_on_athena_response_contains_correct_keys(self): """Test routine exec_sql_on_athena_response_contains_correct_keys""" response = AthenaManager.exec_sql_on_athena(self.athena_manager, self.success_query, self.test_ouput_location) self.assertTrue( 'Status' in response and 'QueryExecutionId' in response and 'Query' in response, 'Response does not include all required keys') def test_on_success_exec_sql_on_athena_retruns_success_status(self): """Test routine on_success_exec_sql_on_athena_retruns_success_status""" response = AthenaManager.exec_sql_on_athena(self.athena_manager, self.success_query, self.test_ouput_location) self.assertTrue(response['Status'] == 'SUCCEEDED', 'SQL command was not successfull') def test_on_failure_exec_sql_on_athena_raises_exception(self): """Test routine on_failure_exec_sql_on_athena_raises_exception""" self.assertRaises( Exception, AthenaManager.exec_sql_on_athena(self.athena_manager, self.fail_query, self.test_ouput_location))
def lambda_handler(event, context): ''' The entry point for the Lambda function :param event: A dictionary containing all the information about the event that triggered the Lambda function :param context: This object contains information about the state of the Lambda function itself. :return: N/A ''' # set environment variables environment_name = event['ResourceProperties']['EnvnameProperty'] micro_environment = event['ResourceProperties']['MicroenvProperty'] client = event['ResourceProperties']['ClientProperty'] account = event['ResourceProperties']['AccountProperty'] ddl_bucket = event['ResourceProperties']['DDLBucketProperty'] ddl_create_key = event['ResourceProperties']['CreateSqlS3KeyProperty'] ddl_delete_key = event['ResourceProperties']['DeleteSqlS3KeyProperty'] data_bucket = event['ResourceProperties']['DataBucketProperty'] test_output_location = event['ResourceProperties'][ 'AthenaSqlOutputUriProperty'] physical_resource_id = context.log_stream_name # Create a dictionary of placeholders and their values that will be replaced in sql commands. dict_of_replacements = { "__bucket__": data_bucket, "__envname__": environment_name, "__microenv__": micro_environment, "__client__": client, "__account__": account } ###Set AWS specific variables### # Instantiate Connection conn = Connection() # Create an Athena connection conn_athena = conn.athena_connection() # Create an S3 connection conn_s3 = conn.s3_connection() # Instantiate S3Manager s3_manager = S3Manager(conn_s3) # Instantiate Athena Manager athena_manager = AthenaManager(conn_athena) try: if event['RequestType'] != 'Delete': # Download file that has the SQL Create Commands from s3 ddl_filename = s3_manager.download_object(ddl_bucket, ddl_create_key) logger.info('Downloaded {}'.format(ddl_filename)) response = execute_scripts(athena_manager, ddl_filename, test_output_location, dict_of_replacements, 'Create') send_response(event, context, SUCCESS, response, physical_resource_id) return response elif event['RequestType'] == 'Delete': # Download file that has the SQL Delete Commands from s3 ddl_filename = s3_manager.download_object(ddl_bucket, ddl_delete_key) logger.info('Downloaded {}'.format(ddl_filename)) response = execute_scripts(athena_manager, ddl_filename, test_output_location, dict_of_replacements, 'Delete') send_response(event, context, SUCCESS, response, physical_resource_id) return response except Exception as error: logger.info(error.args) response_data = {'Reason': error.args} send_response(event, context, FAILED, response_data, physical_resource_id) return error.args
def lambda_handler(event, context): """ This is the main entry point for lambda function :param event: Event object that contains the bucket name and key of the manifest file that triggered the lambda :param context: Contains context information about the lambda function itself e.g. remaining time :return: Does not return anything """ # set environment exec_environment = os.environ['exec_environment'] etl_file_path = os.environ['etl_file_path'] manifest_file_path = os.environ['manifest_file_path'] manifest_file = os.environ['manifest_file'] log_uri = os.environ['log_uri'] # Download manifest file get_manifest_file(event, manifest_file_path, manifest_file) # Instantiate Connection conn = Connection() # Create an EMR connection conn_emr = conn.emr_connection() # Create an S3 connection conn_s3 = conn.s3_connection() # Instantiate S3Manager s3_manager = S3Manager() # Parse the manifest file and generate etl from ETL template wth placeholder values filled in. # Also gets the details about the EMR cluster to create. try: # Instantiate ManifestParser manifest_parser = ManifestParser() logger.info( "Generating new ETL file from ETL template wth placeholder values filled in" ) dest_etl_file = manifest_parser.parse_manifest_file( manifest_file_path, manifest_file, conn_s3, s3_manager, exec_environment) logger.info("Generated: {}".format(dest_etl_file)) except: logger.error( 'Failed while trying to generate a new ETL from s3://{}/{}'.format( manifest_parser.script_s3_bucket, manifest_parser.script_s3_key)) logging.error(sys.exc_info()) raise # Copy generated ETL to S3. This will then be submitted to EMR try: s3_manager.upload_object(conn_s3, etl_file_path, dest_etl_file, manifest_parser.script_s3_bucket, 'generated-etls', dest_etl_file) except: raise # Launch and submit jobs to EMR try: # Instantiate EMR Instance emr = EMRInstance() cluster_name = "{}_{}".format(exec_environment, manifest_parser.script_s3_key) cluster_id = emr.get_first_available_cluster(conn_emr) if manifest_parser.use_existing_cluster and cluster_id: instance_groups = emr.get_instance_groups(conn_emr, cluster_id) group_id = instance_groups['CORE'] instance_groups_count = emr.get_instance_groups_count( conn_emr, cluster_id) current_instance_count = instance_groups_count[group_id] if manifest_parser.instance_count > current_instance_count: emr.set_instance_count(conn_emr, cluster_id, group_id, manifest_parser.instance_count) # Allow 10 secs for resizing to start time.sleep(10) #submit job emr.submit_job( conn_emr, cluster_id, 's3://{}/generated-etls/{}'.format( manifest_parser.script_s3_bucket, dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE') else: # Launch EMR cluster emr.launch_emr_and_submit_job( conn_emr, log_uri, 's3://{}/generated-etls/{}'.format( manifest_parser.script_s3_bucket, dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE', '{}'.format(cluster_name), manifest_parser.terminate_cluster, manifest_parser.instance_type, manifest_parser.instance_count) logger.info("Submitted s3://{}/{} to process_{}".format( manifest_parser.script, dest_etl_file, cluster_name)) except: logger.error( "Failed while trying to launch EMR cluster. Details below:") raise
#!/usr/bin/Python # IMPORT classes from aws package from aws import Connection from aws import EC2Instance from aws import Volumes # INSATANTIATE Connection connInst = Connection() # CREATE an EC2 Connection reg = connInst.list_regions() print reg for regionaws in reg: #rgname = reg #print rgname # return rgname conn = connInst.ec2Connection(regionaws) # PRINT connection print conn # Instantiate EC2Instance for operations # ec2 = EC2Instance() # InsID = ec2.list_instances(conn) # print InsID # for Ins in InsID: # Ins.start() # print (Ins, "is starting now") # ec2 = EC2Instance()
#!/usr/bin/Python #Import classes from aws package from aws import Connection from aws import EC2Instance from aws import Volumes from aws import Security connInst = Connection() conn = connInst.ec2Connection() ''' Uncomment code as required. Replace instance_id with actual values ''' #Manage EC2 Instances ''' instance = EC2Instance() instance.list_instances(conn) instance.start_instance(conn, instance_id) instance.stop_instance(conn, instance_id) ''' ''' Uncomment code as required. Replace instance_id and volume_id with actual values ''' #Manage EBS Volumes ''' volumeInst = Volumes() volumeInst.list_volumes(conn) volumeInst.detach_volume(conn,volume_id) volumeInst.attach_volume(conn, volume_id, instance_id)
class TestSqlDeployCustomResourceLambda(unittest.TestCase): def setUp(self): """Setup""" test_name = self.shortDescription() # Instantiate Connection self.conn = Connection() # Create an Athena connection self.conn_athena = self.conn.athena_connection() # Instantiate Athena Manager self.athena_manager = AthenaManager(self.conn_athena) self.test_ouput_location = 's3://aws-athena-query-results-933886674506-eu-west-1/' if test_name == "Test routine get_sql_commands_from_file": self.test_list_of_commands = [ 'CREATE DATABASE IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS unittest' ] if test_name == "Test routine replace_placeholders_in_sql": self.test_list_of_commands = [ 'CREATE __client__ IF NOT EXISTS __bucket__', 'DROP DATABASE IF EXISTS __client_____bucket__' ] self.test_dict_of_replacements = { '__bucket__': 'unittest', '__client__': "batman" } self.test_list_of_updated_commands = [ 'CREATE batman IF NOT EXISTS unittest', 'DROP DATABASE IF EXISTS batman_unittest' ] def test_get_sql_commands_from_file(self): """Test routine get_sql_commands_from_file""" list_of_commands = s.get_sql_commands_from_file('test_split.txt', ';') self.assertEqual(self.test_list_of_commands, list_of_commands, "Lists of commands don't match") def test_execute_scripts_with_create_event(self): """Test routine execute_scripts""" dict_of_responses = s.execute_scripts(self.athena_manager, 'test_split.txt', self.test_ouput_location, 'Create') self.assertTrue( dict_of_responses['Create'][0]['Status'] == 'SUCCEEDED', 'First command failed.') self.assertTrue( dict_of_responses['Create'][1]['Status'] == 'SUCCEEDED', 'Second command failed.') def test_execute_scripts_with_delete_event(self): """Test routine execute_scripts""" dict_of_responses = s.execute_scripts(self.athena_manager, 'test_split.txt', self.test_ouput_location, 'Delete') self.assertTrue( dict_of_responses['Delete'][0]['Status'] == 'SUCCEEDED', 'First command failed.') self.assertTrue( dict_of_responses['Delete'][1]['Status'] == 'SUCCEEDED', 'Second command failed.') def test_replace_placeholders_in_sql(self): """Test routine replace_placeholders_in_sql""" list_of_updated_commands = s.replace_placeholders_in_sql( self.test_list_of_commands, self.test_dict_of_replacements) self.assertEqual(list_of_updated_commands, self.test_list_of_updated_commands, "The placeholder replacement failed.")