def dailyBatchUpload(file, exchange):
"""
:param 1: 'Historical/NASDAQ/{file}'
1. Import new daily csv
2. Parse Date
3. Format Date
4. Save df without index and header
5. Append to remote RDS or local DB (change param). Default = RDS
"""
if "NASDAQ" in file:
df = pd.read_csv(f'Historical/{exchange}/Already Uploaded/{file}')
else:
df = pd.read_csv(f'Historical/{exchange}/Already Uploaded/{file}')
df = dateParsing(df)
df = dateFormat(df)
if "NASDAQ" in file:
dfToRDS(df=df, table='NASDAQ_15', db_name='marketdata', location='RDS')
else:
dfToRDS(df=df, table='NYSE_15', db_name='marketdata', location='RDS')
return df
def sendSingleRDS():
df = pd.read_csv('marketdata.csv')
df = df.iloc[:, 1:-1]
df = df.rename(columns={
"Aroon Down": "Aroon_Down",
"Aroon Up": "Aroon_Up"
})
dfToRDS(df=df, table='Signals_details', db_name='signals')
def csvToRDS():
df = pd.read_csv(f'utils/batch_{today_str}.csv')
# dataFrame.iloc[<ROWS INDEX RANGE> , <COLUMNS INDEX RANGE>]
df = df.iloc[:, 1::]
dfToRDS(df=df, table='Signals_aroon_crossing', db_name='signals')
db_acc_obj.exc_query(db_name='signals', query=quCopy)
db_acc_obj.exc_query(db_name='signals', query=quDeletePreviousTable)
db_acc_obj.exc_query(db_name='signals', query=quRenameTable)
def sendDataInChunks(finalDF):
"""
Sending the data by chunks otherwise EC2 --> RDS conn. stops for obscure reasons.
Sleep seems also necessary. 5 secs seems ok.
"""
lenDF = len(finalDF)
nChunks = round(lenDF / 50000)
initChunk = True
if nChunks > 0:
for i in list(range(nChunks)):
if initChunk == True:
chunk = finalDF[0:50000]
initChunk = False
currentChunk = 50000
print("Sending chunk n°", i)
dfToRDS(df=chunk, table='Signals_details', db_name='signals')
time.sleep(5)
print('Next chunk')
else:
nextChunk = currentChunk + 50000
chunk = finalDF[currentChunk:nextChunk]
currentChunk = nextChunk
print("Sending chunk n°", i)
dfToRDS(df=chunk, table='Signals_details', db_name='signals')
time.sleep(5)
if i < nChunks:
print('Next chunk')
else:
dfToRDS(df=finalDF, table='Signals_details', db_name='signals')
print("To RDS: Completed!")
def sendData(self):
dfToRDS(df=self.df, table='Technicals', db_name='marketdata')
ON Signals_aroon_crossing.ValidTick = t.Symbol\
)t2;"
quPull = "SELECT * FROM signals.Signals_aroon_crossing_close"
db_acc_obj.exc_query(db_name='marketdata', query=squDelPrevDaysInUSTODAY)
db_acc_obj.exc_query(db_name='signals', query=squDelPreviousTableEvol)
db_acc_obj.exc_query(db_name='signals', query=squDelPreviousTableClose)
db_acc_obj.exc_query(db_name='signals', query=quCreate)
items = db_acc_obj.exc_query(db_name='signals', query=quPull, \
retres=QuRetType.ALLASPD)
items['PriceEvolution'] = ((
(items.iloc[:, 5] - items.iloc[:, 4]) / items.iloc[:, 4]) *
100).tolist()
items = items.rename(columns={"Close": "LastClosingPrice"})
return items
if __name__ == '__main__':
db_acc_obj = std_db_acc_obj()
print('Calculting price evolutions. . .')
items = signalsPricesEvol()
print('Sending data to RDS. . .')
dfToRDS(df=items,
table='Signals_aroon_crossing_evol',
db_name='signals',
location='RDS')