#Constants for Data Extraction
LOGIN_URL = 'https://www.lendingclub.com/auth/login'
DOWNLOAD_URL = 'https://www.lendingclub.com/info/download-data.action'
DIR_PATH = 'Data\DOWNLOAD_LOAN_DATA'
#Parameters for lending club data scraping
fileTag = "loanStatsFileNamesJS"
# email= os.environ['LENDING_CLUB_EMAIL']
# password= os.environ['LENDING_CLUB_PASSWORD']
print('Downloading Files...')
#Extract Data from Lending Club URL
de = DataExtractor(email, password)
de.extractData(LOGIN_URL=LOGIN_URL,
DOWNLOAD_URL=DOWNLOAD_URL,
fileTag=fileTag)
print('Ingesting Data...')
#Ingest Data into Pipeline
di = DataIngestor(aws_client)
#Create Landing and Processed Buckets
LANDING_BUCKET = 'lending-club-landing-data'
PROCESSED_BUCKET = 'lending-club-processed-data'
print('Creating Buckets...')
di.createS3Bucket(LANDING_BUCKET)
di.createS3Bucket(PROCESSED_BUCKET)
print('Uploading Files...')
#!/usr/bin/python import sys from DataExtractor import DataExtractor deExtractor = DataExtractor() deExtractor.extractData(sys.argv[1])
#!/usr/bin/python import sys from DataExtractor import DataExtractor deExtractor = DataExtractor() deExtractor.extractData(sys.argv[1])
