def SendLiveArbitrageDataAllTickers():
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
try:
live_data = PerMinDataOperations().LiveFetchPerMinArbitrage()
live_data = live_data[[
'symbol', 'Arbitrage in $', 'ETF Trading Spread in $', 'ETF Price',
'ETF Change Price %', 'Net Asset Value Change%', 'ETFMover%1',
'ETFMover%2', 'ETFMover%3', 'ETFMover%4', 'ETFMover%5', 'Change%1',
'Change%2', 'Change%3', 'Change%4', 'Change%5', 'Timestamp'
]]
live_data = OverBoughtBalancedOverSold(df=live_data)
live_data.rename(
columns={'Magnitude of Arbitrage': 'Absolute Arbitrage'},
inplace=True)
live_data = live_data.round(3)
live_data = live_data.fillna(0)
return jsonify(live_data.to_dict(orient='records'))
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def TradePricesForTickers(self, start_dt_ts, end_dt_ts):
TradesDataCursor = PerMinDataOperations().FetchAllTradeDataPerMin(
start_dt_ts, end_dt_ts)
TradePriceDf = pd.DataFrame(list(TradesDataCursor))
TradePriceDf = TradePriceDf.assign(
price=(TradePriceDf['h'] + TradePriceDf['l']) /
2).drop(columns=['h', 'l'])
TradePriceDf.set_index('sym', inplace=True)
return TradePriceDf
def live_arb_candlestick_and_signal_categorization(etfname, bypass_auth=False):
if not bypass_auth:
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
else:
pass
try:
PerMinObj = PerMinDataOperations()
res = fecthArbitrageANDLivePrices(
etfname=etfname,
FuncETFPrices=PerMinObj.FetchFullDayPricesForETF,
FuncArbitrageData=PerMinObj.FetchFullDayPerMinArbitrage,
callAllDayArbitrage=True)
if type(res) == Response:
return res
last_minute = res['Arbitrage']['Time'].tail(1).values[0]
signals_dict = analyzeSignalObj.analyze_etf_for_all_patterns(
res['Arbitrage'])
list(
map(
lambda x: signals_dict.update(
{x: ('No Occurrence yet', 'No Signal')})
if x not in signals_dict else None, CandlesignalsColumns))
signals_dict = [[x.replace(' ', ''), *v]
for x, v in signals_dict.items()]
last_minute_signal = list(
itertools.filterfalse(lambda x: last_minute not in x,
signals_dict))
last_minute_signal = " ".join(last_minute_signal[0]) if len(
last_minute_signal) > 0 else "No Pattern"
pricedf = res['Prices']
pricedf = pricedf.reset_index(drop=True)
pricedf['Time'] = pricedf['date']
pricedf['Time'] = pricedf['Time'].apply(lambda x: str(x.time()))
pricedf = pd.merge(res['Arbitrage'][['Time', 'Over Bought/Sold']],
pricedf,
on='Time',
how='right')
pricedf = pricedf[pricedf['Over Bought/Sold'].notna()]
del pricedf['Time']
res['Prices'] = pricedf
res['Prices'] = res['Prices'].to_csv(sep='\t', index=False)
res['SignalCategorization'] = CategorizeSignals(
ArbitrageDf=res['Arbitrage'],
ArbitrageColumnName='Arbitrage in $',
PriceColumn='ETF Price',
Pct_change=True)
res['CandlestickSignals'] = signals_dict
res['last_minute_signal'] = last_minute_signal
res.pop('Arbitrage')
return jsonify(res)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def on_message(ws, message):
start = time.time()
responses = ujson.loads(message)
#dataQ = [response for response in responses if response['ev'] == 'Q']
dataAM = [response for response in responses if response['ev'] == 'AM']
if dataAM:
loop = asyncio.get_event_loop()
loop.run_until_complete(PerMinDataOperations().do_insert(dataAM))
logger.debug("Aggregates-Minute Inserted")
'''
if dataQ:
print("Quotes")
print(dataQ)
PerMinDataOperations().insertQuotesLive(dataQ)
'''
end = time.time()
def fetch_save_Last_Quotes(self):
try:
#print("Fetching")
responseData = self.getDataFromPolygon(
methodToBeCalled=self.extractQuotesDataFromResponses,
getUrls=self.getUrls)
responseData = list(responseData)
#print("Inserting")
#print(datetime.datetime.now())
#print(responseData)
PerMinDataOperations().insertQuotesLive(quotesData=responseData)
#print("Inserted Successfully")
logger.debug("Quotes Inserted")
except Exception as e:
#print("Insertion failed")
logger.exception(e)
pass
def UpdateLiveArbitrageDataTablesAndPrices(etfname, bypass_auth=False):
if not bypass_auth:
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
else:
pass
try:
PerMinObj = PerMinDataOperations()
res = fecthArbitrageANDLivePrices(
etfname=etfname,
FuncETFPrices=PerMinObj.LiveFetchETFPrice,
FuncArbitrageData=PerMinObj.LiveFetchPerMinArbitrage,
callAllDayArbitrage=False)
if type(res) == Response:
return res
etfmoversDictCount, highestChangeDictCount = etfMoversChangers(
res['Arbitrage'])
res['etfmoversDictCount'] = etfmoversDictCount
res['highestChangeDictCount'] = highestChangeDictCount
res['Arbitrage'] = OverBoughtBalancedOverSold(df=res['Arbitrage'])
arbitrage_columns = list(res['Arbitrage'].columns)
res['Arbitrage'].rename(columns={
x: x.replace(' ', '_')
for x in arbitrage_columns if ' ' in x
},
inplace=True)
res['Prices'] = res['Prices'].to_dict(orient='records')[0]
res['Arbitrage'] = res['Arbitrage'].to_dict(orient='records')[0]
res['SignalInfo'] = analyzeSignalPerformane(
res['Arbitrage']['Arbitrage_in_$'])
return res
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def SendLiveArbitrageDataSingleTicker(etfname, bypass_auth=False):
if not bypass_auth:
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
else:
pass
try:
PerMinObj = PerMinDataOperations()
res = fecthArbitrageANDLivePrices(
etfname=etfname,
FuncETFPrices=PerMinObj.FetchFullDayPricesForETF,
FuncArbitrageData=PerMinObj.FetchFullDayPerMinArbitrage,
callAllDayArbitrage=True)
if type(res) == Response:
return res
last_minute = res['Arbitrage']['Time'].tail(1).values[0]
signals_dict = analyzeSignalObj.analyze_etf_for_all_patterns(
res['Arbitrage'])
list(
map(
lambda x: signals_dict.update(
{x: ('No Occurrence yet', 'No Signal')})
if x not in signals_dict else None, CandlesignalsColumns))
signals_dict = [[x.replace(' ', ''), *v]
for x, v in signals_dict.items()]
last_minute_signal = list(
itertools.filterfalse(lambda x: last_minute not in x,
signals_dict))
last_minute_signal = " ".join(last_minute_signal[0]) if len(
last_minute_signal) > 0 else "No Pattern"
pricedf = res['Prices']
pricedf = pricedf.reset_index(drop=True)
pricedf['Time'] = pricedf['date']
pricedf['Time'] = pricedf['Time'].apply(lambda x: str(x.time()))
pricedf = pd.merge(res['Arbitrage'][['Time', 'Over Bought/Sold']],
pricedf,
on='Time',
how='right')
pricedf = pricedf[pricedf['Over Bought/Sold'].notna()]
del pricedf['Time']
res['Prices'] = pricedf
res['Prices'] = res['Prices'].to_csv(sep='\t', index=False)
res['pnlstatementforday'] = res['pnlstatementforday']
res['SignalCategorization'] = CategorizeSignals(
ArbitrageDf=res['Arbitrage'],
ArbitrageColumnName='Arbitrage in $',
PriceColumn='ETF Price',
Pct_change=True)
etfmoversDictCount, highestChangeDictCount = etfMoversChangers(
res['Arbitrage'])
res['etfmoversDictCount'] = etfmoversDictCount
res['highestChangeDictCount'] = highestChangeDictCount
res['scatterPlotData'] = res['Arbitrage'][[
'ETF Change Price %', 'Net Asset Value Change%'
]].to_dict(orient='records')
res['ArbitrageLineChart'] = res['Arbitrage'][[
'Arbitrage in $', 'Time'
]].to_dict('records')
res['SignalInfo'] = analyzeSignalPerformane(
res['Arbitrage'].tail(1).to_dict('records')[0]['Arbitrage in $'])
arbitrage_columns = list(res['Arbitrage'].columns)
res['Arbitrage'].rename(
columns={x: x.replace(' ', '_')
for x in arbitrage_columns},
inplace=True)
res['Arbitrage'] = res['Arbitrage'].fillna(0)
res['Arbitrage'] = res['Arbitrage'].to_dict(orient='records')
res['CandlestickSignals'] = signals_dict
res['last_minute_signal'] = last_minute_signal
return jsonify(res)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def __init__(self):
self.perMinDataObj = PerMinDataOperations()
self.spreadDF = pd.DataFrame()
class PerMinAnalysis():
def __init__(self):
self.perMinDataObj = PerMinDataOperations()
self.spreadDF = pd.DataFrame()
def get_ts_for_fetching_data(self):
#######################################################
# UTC Timestamps for pulling data from QuotesLiveData DB, below:
#######################################################
end_dt = datetime.datetime.now().replace(second=0, microsecond=0)
end_dt_ts = int(end_dt.timestamp() * 1000)
start_dt = end_dt - datetime.timedelta(minutes=1)
start_dt_ts = int(start_dt.timestamp() * 1000)
return end_dt, end_dt_ts, start_dt, start_dt_ts
def calculate_spread_for_minute(self):
#######################################################
# ETF Spread Calculation
#######################################################
try:
timestamps = self.get_ts_for_fetching_data()
end_dt_ts = timestamps[1]
start_dt_ts = timestamps[3]
QuotesResultsCursor = self.perMinDataObj.FetchQuotesLiveDataForSpread(
start_dt_ts, end_dt_ts)
QuotesDataDf = pd.DataFrame(list(QuotesResultsCursor))
QuotesDataDf['ETF Trading Spread in $'] = QuotesDataDf[
'askprice'] - QuotesDataDf['bidprice']
self.spreadDF = QuotesDataDf.groupby(['symbol']).mean()
except Exception as e:
traceback.print_exc()
logger.exception(e)
for col in self.spreadDF.columns:
self.spreadDF[col].values[:] = 0
pass
def PerMinAnalysisCycle(self, obj):
try:
#######################################################
# UTC Timestamps for pulling data from QuotesLiveData DB, below:
#######################################################
timestamps = self.get_ts_for_fetching_data()
end_dt_ts = timestamps[1]
start_dt = timestamps[2]
start_dt_ts = timestamps[3]
#######################################################
# ETF Arbitrage Calculation
#######################################################
startarb = time.time()
arbDF = obj.calcArbitrage(end_dt_ts=end_dt_ts,
start_dt_ts=start_dt_ts,
start_dt=start_dt)
#######################################################
# Results:
#######################################################
mergeDF = arbDF.merge(self.spreadDF,
how='outer',
left_index=True,
right_index=True)
mergeDF.reset_index(inplace=True)
mergeDF.rename(columns={"index": "Symbol"}, inplace=True)
cols = list(mergeDF.columns)
cols = [cols[0]] + [cols[-1]] + cols[1:-1]
mergeDF = mergeDF[cols]
mergeDF['Timestamp'] = start_dt_ts
SaveCalculatedArbitrage().insertIntoPerMinCollection(
start_ts=start_dt_ts,
ArbitrageData=mergeDF.to_dict(orient='records'))
logger.debug("arbDF")
logger.debug(arbDF)
logger.debug("spreadDF")
logger.debug(self.spreadDF)
logger.debug("mergeDF")
logger.debug(mergeDF)
except Exception as e:
traceback.print_exc()
logger.exception(e)
# emailobj = EmailSender()
# msg = emailobj.message(subject=e,
# text="Exception Caught in ETFLiveAnalysisProdWS/PerMinCaller.py {}".format(
# traceback.format_exc()))
# emailobj.send(msg=msg, receivers=['*****@*****.**', '*****@*****.**'])
pass