def run_etl():
'''
- Connects to the Redshift database
- Loads staging tables to be used to further processing
- Processes the staging tables and loads them into the final star schema
'''
config = configparser.ConfigParser()
CONFIG_FILE = 'dwh.cfg'
config.read(CONFIG_FILE)
DWH_DB = config.get("CLUSTER", "DB_NAME")
DWH_DB_USER = config.get("CLUSTER", "DB_USER")
DWH_DB_PASSWORD = config.get("CLUSTER", "DB_PASSWORD")
DWH_PORT = config.get("CLUSTER", "DB_PORT")
aws_manager = AWSManager()
HOST = aws_manager.get_cluster_endpoint()
conn = psycopg2.connect(
f"host={HOST} dbname={DWH_DB} user={DWH_DB_USER} password={DWH_DB_PASSWORD} port={DWH_PORT}"
)
cur = conn.cursor()
print("Loading Staging tables. Please wait...")
load_staging_tables(cur, conn)
print("Finished!\n")
print("Loading dimensional tables. Please wait...")
insert_tables(cur, conn)
print("Finished!\n")
conn.close()
def __init__(self, record_training_data=False, model=None):
print('Setting up PiCar...')
if record_training_data and not model:
self.record_training_data = True
print('Recording training data...')
else:
self.record_training_data = False
if model:
self.model_manager = ModelManager()
self.model_manager.load_model(model)
else:
self.model_manager = None
self.dc = DriveController()
self.aws_manager = AWSManager()
self.setup_camera()
# Structs for collecting training images and labels
self.training_images = []
self.training_labels = []
# Stores the current user drive instruction
self.current_drive_input = 'forward'
# Using pygame in process remote keyboard inputs
pygame.init()
pygame.display.set_mode((100, 100))
def run_initial_setup():
'''
- Drops all tables in the Redshift database so that the ETL script can be rerun.
- Creates all Redshift tables.
'''
config = configparser.ConfigParser()
CONFIG_FILE = 'dwh.cfg'
config.read(CONFIG_FILE)
DWH_DB = config.get("CLUSTER", "DB_NAME")
DWH_DB_USER = config.get("CLUSTER", "DB_USER")
DWH_DB_PASSWORD = config.get("CLUSTER", "DB_PASSWORD")
DWH_PORT = config.get("CLUSTER", "DB_PORT")
aws_manager = AWSManager()
HOST = aws_manager.get_cluster_endpoint()
conn = psycopg2.connect(
f"host={HOST} dbname={DWH_DB} user={DWH_DB_USER} password={DWH_DB_PASSWORD} port={DWH_PORT}"
)
cur = conn.cursor()
print("Resetting Tables.")
drop_tables(cur, conn)
create_tables(cur, conn)
print("Finished!\n")
conn.close()
def set_up_aws_resources():
'''
- Creates the required IAM role to access data from S3
- Creates a new Redshift Cluster
'''
aws_manager = AWSManager()
print("Creating IAM role")
aws_manager.create_iam_role()
print("Attaching policy to IAM role\n")
aws_manager.attach_policy()
print("Creating Redshift Cluster. This might take a few minutes...")
aws_manager.create_cluster()
aws_manager.wait_for_cluster_creation()
print("Success! Cluster created.\n")
def tear_down_resources():
'''
Cleanup phase - tears down all create AWS resources during this ETL process.
'''
aws_manager = AWSManager()
print("Deleting IAM Role & Attached Policy.")
aws_manager.delete_iam_role()
print("Deleting Cluster. This may take a few minutes to reflect in your account.")
aws_manager.delete_cluster()
import configparser
import boto3
from aws_manager import AWSManager
# CONFIG
config = configparser.ConfigParser()
CONFIG_FILE = 'dwh.cfg'
config.read(CONFIG_FILE)
aws_manager = AWSManager()
REGION_NAME = aws_manager.get_region_name()
IAM_ROLE_NAME = aws_manager.get_cluster_iam_role()
LOG_DATA = config.get("S3", "LOG_DATA")
LOG_JSONPATH = config.get("S3", "LOG_JSONPATH")
SONG_DATA = config.get("S3", "SONG_DATA")
# DROP TABLES
staging_events_table_drop = "DROP TABLE IF EXISTS staging_events"
staging_songs_table_drop = "DROP TABLE IF EXISTS staging_songs"
songplay_table_drop = "DROP TABLE IF EXISTS factSongplay"
user_table_drop = "DROP TABLE IF EXISTS dimUser"
song_table_drop = "DROP TABLE IF EXISTS dimSong"
artist_table_drop = "DROP TABLE IF EXISTS dimArtist"
time_table_drop = "DROP TABLE IF EXISTS dimTime"
# CREATE TABLES
staging_events_table_create = ("""CREATE TABLE IF NOT EXISTS staging_events
(artist varchar,
class PiCar:
# Camera Info
SCREEN_WIDTH = 200
SCREEN_HEIGHT = 66
FRAME_RATE = 24
def __init__(self, record_training_data=False, model=None):
print('Setting up PiCar...')
if record_training_data and not model:
self.record_training_data = True
print('Recording training data...')
else:
self.record_training_data = False
if model:
self.model_manager = ModelManager()
self.model_manager.load_model(model)
else:
self.model_manager = None
self.dc = DriveController()
self.aws_manager = AWSManager()
self.setup_camera()
# Structs for collecting training images and labels
self.training_images = []
self.training_labels = []
# Stores the current user drive instruction
self.current_drive_input = 'forward'
# Using pygame in process remote keyboard inputs
pygame.init()
pygame.display.set_mode((100, 100))
def setup_camera(self):
print('Initializing camera...')
self.camera = PiCamera()
self.camera.resolution = (self.SCREEN_WIDTH, self.SCREEN_HEIGHT)
self.camera.framerate = self.FRAME_RATE
self.raw_capture = PiRGBArray(self.camera, size=(self.SCREEN_WIDTH, self.SCREEN_HEIGHT))
# allow the camera to warm up
time.sleep(1)
print('Camera initialization complete...')
def process_user_inputs(self):
for event in pygame.event.get():
if event.type == pygame.KEYUP:
self.dc.forward()
self.current_drive_input = 'forward'
elif event.type == pygame.KEYDOWN:
if event.key == pygame.K_RIGHT:
self.dc.pivot_right()
self.current_drive_input = 'right'
elif event.key == pygame.K_LEFT:
self.dc.pivot_left()
self.current_drive_input = 'left'
else:
self.dc.forward()
self.current_drive_input = 'forward'
def convert_model_output_to_drive_command(self, model_output):
print(str(model_output))
max_index = np.argmax(model_output)
if max_index == 0:
self.dc.forward()
self.current_drive_input = 'forward'
elif max_index == 1:
self.dc.pivot_right()
self.current_drive_input = 'right'
elif max_index == 2:
self.dc.pivot_left()
self.current_drive_input = 'left'
print(self.current_drive_input)
def drive(self):
for frame in self.camera.capture_continuous(self.raw_capture, format="bgr", use_video_port=True):
image = frame.array
if self.model_manager:
# Use model to steer PiCar
image = image / 255.0
output = self.model_manager.run_inference(image)
self.convert_model_output_to_drive_command(output)
else:
# Use remote keyboard inputs to steer PiCar
self.process_user_inputs()
# Record training data
if self.record_training_data:
self.training_images.append(image)
self.training_labels.append(self.current_drive_input)
# Display camera feed
cv2.imshow("Feed", image)
key = cv2.waitKey(1) & 0xFF
self.raw_capture.truncate(0)
# Exit program if user pressed 'q'
if key == ord("q"):
self.dc.stop()
# Upload training data to AWS before exiting
if self.record_training_data:
self.aws_manager.upload_training_data(self.training_images, self.training_labels)
break