def __init__(self):
self.config = config.Config()
self.cursor_factory = psycopg2.extras.RealDictCursor
self.delimiter = "\t"
self.aws_access_key_id = self.config.aws_access_key_id
self.aws_secret_access_key = self.config.aws_secret_access_key
self.ignore_header = " IGNOREHEADER 1 "
def __init__(self):
self.config = config.Config()
self.db = MySQLdb.connect(
host=self.config.mysql_params['DB_HOST'],
db=self.config.mysql_params['DB_DATABASE'],
passwd=self.config.mysql_params['DB_PASSWORD'],
user=self.config.mysql_params['DB_USERNAME'],
cursorclass=MySQLdb.cursors.DictCursor)
self.cursor = self.db.cursor()
filterwarnings('ignore', category=MySQLdb.Warning)
def __init__(self):
self.config = config.Config()
self.helpers = helpers.Helpers()
self.merged_customers = merged_customers.MergedCustomers()
self.merged_dict = self.merged_customers.get_merged_dict()
# Clean up
del (self.merged_customers)
self.insert_customer_rows = insert_customer_rows.InsertCustomerRows()
self.insert_customer_info_rows = insert_customer_info_rows.InsertCustomerInfoRows(
)
def __init__(self):
self.config = config.Config()
self.ftp_creds = self.config.ftp_creds
self.archive_bucket = self.config.archive_bucket
self.dealervault_filelist = []
self.archive_filelist = []
self.missing_files = {}
self.ftp_wrapper = ftp_wrapper.FtpWrapper(self.ftp_creds)
self.s3_wrapper = s3_wrapper.S3Wrapper()
self.s3_wrapper.setBucket(self.archive_bucket)
def __init__(self):
logging.info('initializing constructor')
self.config = config.Config()
self.get_all_customers = get_all_customers.GetAllCustomers()
self.get_all_vehicles = get_all_vehicles.GetAllVehicles()
self.deduper = deduper.Deduper()
self.dedupe_vehicles = dedupe_vehicles.DedupeVehicles()
self.customer_dealer_insertion = customer_dealer_insertion.CustomerDealerInsertion(
)
self.vehicle_insertion = vehicle_insertion.VehicleInsertion()
# Get Missing Timestamps (a callable class that is a generator)
self.missing_timestamps = missing_timestamps.MissingTimestamps()
def __init__(self):
self.timestamp = int(time.time())
self.config = config.Config()
self.add_columns = add_columns.AddColumns(self.timestamp)
self.archive_bucket = self.config.archive_bucket
self.processed_archive_bucket = self.config.processed_archive_bucket
self.s3_archive_wrapper = s3_wrapper.S3Wrapper()
self.s3_archive_wrapper.setBucket(self.archive_bucket)
self.s3_processed_wrapper = s3_wrapper.S3Wrapper()
self.s3_processed_wrapper.setBucket(self.processed_archive_bucket)
self.archive_filelist = []
self.processed_filelist = []
self.missing_files = []
def __init__(self):
self.config = config.Config()
self.processed_archive_bucket = self.config.processed_archive_bucket
self.new_import_bucket = self.config.processed_new_imports
self.s3_processed_wrapper = s3_wrapper.S3Wrapper()
self.s3_processed_wrapper.setBucket(self.processed_archive_bucket)
self.s3_newimport_wrapper = s3_wrapper.S3Wrapper()
self.s3_newimport_wrapper.setBucket(self.new_import_bucket)
self.uploaded_files = uploaded_files.UploadedFiles()
self.redshift_copy = redshift_copy.RedshiftCopy()
self.redshift_insert = redshift_insert.RedshiftInsert()
self.all_timestamps = all_timestamps.AllTimestamps()
self.processed_filelist = []
self.previously_uploaded_files = []
self.missing_files = []
self.timestamp = 0
def __init__(self):
logging.info('initializing materializer')
self.config = config.Config()
self.mysql_connection = mysql_connection.MysqlConnection()
# Get Missing Timestamps (a callable class that is a generator)
logging.info('getting all missing_timestamps')
self.missing_timestamps = missing_timestamps.MissingTimestamps()
logging.info(
'initializing table_queries (get_all_rows_sale, get_all_rows_service)'
)
self.table_queries = {
'sales': get_all_rows_sales.GetAllRowsSales(),
'service': get_all_rows_service.GetAllRowsService()
}
logging.info('initializing subs')
self.subs = [
sub_dealers.SubDealers(),
sub_vehicles.SubVehicles(),
sub_customers.SubCustomers()
]
def __init__(self):
self.config = config.Config()
self.merged_dict = {}
self.api_creds = self.config.api_creds
self.api_urls = self.config.api_urls
def __init__(self):
self.config = config.Config()
self.helpers = helpers.Helpers()
self.insert_vehicle_rows = insert_vehicle_rows.InsertVehicleRows()
def __init__(self):
self.config = config.Config()
self.cursor_factory = psycopg2.extras.RealDictCursor
def __init__(self):
self.config = config.Config()
self.aws_access_key_id = self.config.aws_access_key_id
self.aws_secret_access_key = self.config.aws_secret_access_key
#!/usr/bin/env python3
import sys
import packages.archiver as archiver
import packages.processor as processor
import packages.importer as importer
import packages.constructor as constructor
import packages.materializer as materializer
import bugsnag
import packages.config as config
import logging
import os
config = config.Config()
bugsnag.configure(api_key=config.bugsnag_creds['api_key'],
project_root=config.bugsnag_creds['project_root'])
try:
os.remove(config.logging_file)
except OSError:
pass
logging.basicConfig(filename=config.logging_file, level=logging.DEBUG)
if len(sys.argv) == 1 or sys.argv[1] \
not in ['archive', 'process', 'import', 'construct', 'materialize']:
print("USE app.py [archive|process|import|construct|materialize]")
sys.exit()
i = 1