def __init__(self,
date=None,
end_time='21:00:00',
end_time_loop='20:00:00',
polygon_method=None,
symbol_status=None,
collection_name=None,
insert_into_collection=None):
self.helperObj = Helper()
self.date = date
self.extract_data_till_time = self.helperObj.stringTimeToDatetime(
date=date, time=end_time_loop)
self.end_ts = self.helperObj.convertHumanTimeToUnixTimeStamp(
date=date, time=end_time)
self.polygon_method = polygon_method
self.insert_into_collection = insert_into_collection
self.collection_name = collection_name
self.symbol_status = symbol_status
def fetchOHLCHistoricalData(etfname=None, StartDate=None):
ob = HistoricOHLC()
data = ob.getopenlowhistoric(etfname=etfname, startdate=StartDate)
del data['n']
data.rename(columns={
'v': 'volume',
'o': 'open',
'c': 'close',
'h': 'high',
'l': 'low',
't': 'date'
},
inplace=True)
# Helper Class from PolygonTickData.Helper for converting uni timestamp to human timestamp
helperObjTimeConversion = Helper()
data['date'] = data['date'].apply(
lambda x: helperObjTimeConversion.getHumanTime(ts=x, divideby=1000))
return data
def __init__(self, symbols=None, date=None):
self.date = date
self.endTs = Helper().convertHumanTimeToUnixTimeStamp(date=self.date,
time='17:00:00')
self.symbols = symbols
self.mtqd = MongoTradesQuotesData()
from datetime import datetime, timedelta, date
from MongoDB.MongoDBConnections import MongoDBConnectors
from CalculateETFArbitrage.Helpers.LoadEtfHoldings import LoadHoldingsdata
from CommonServices.Holidays import LastWorkingDay
from PolygonTickData.Helper import Helper
from dataclasses import dataclass
sys_private_ip = socket.gethostbyname(socket.gethostname())
if sys_private_ip == '172.31.76.32':
connection = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
per_sec_trades_db = connection.ETF_db.PerSecLiveTrades
per_sec_quotes_db = connection.ETF_db.PerSecLiveQuotes
helper_object = Helper()
class TradeStruct(
): # For Trade Objects, containing current minute and last minute price for Tickers
__slots__ = ['symbol', 'priceT', 'priceT_1', 'price_pct_chg']
def calc_pct_chg(self, priceT, priceT_1):
if priceT_1 == 0:
return 0
return ((priceT - priceT_1) / priceT_1) * 100
def __init__(self, symbol, priceT, priceT_1=None):
self.symbol = symbol
self.priceT = priceT
if not priceT_1:
def fecthArbitrageANDLivePrices(etfname=None,
FuncETFPrices=None,
FuncArbitrageData=None,
callAllDayArbitrage=None):
try:
# Full day historical Prie for ETF
PriceDF = FuncETFPrices(etfname)
#PriceDF['Price'] = (PriceDF['high']+PriceDF['low'])/2
# Full day historical Arbitrage for ETF
ArbitrageDFSemi = FuncArbitrageData(etfname=etfname)
# noh = list(MongoDBConnectors().get_pymongo_readonly_devlocal_production().ETF_db.ETFHoldings.find({'ETFTicker':etfname},{'_id':0, 'NumberOfHolding':1}).sort([('FundHoldingsDate',-1)]).limit(1))[0]['NumberOfHolding']
# x = noh if noh<10 else 10
# etmoverslist = ['ETFMover%{}'.format(i+1) for i in range(x)]
# changes = ['Change%{}'.format(i+1) for i in range(x)]
# etmoverslist.extend(changes)
ArbitrageDf = ArbitrageDFSemi.merge(PriceDF,
left_on='Timestamp',
right_on='date',
how='left')
ArbitrageDf = ArbitrageDf[[
'symbol', 'Timestamp', 'Arbitrage in $', 'ETF Trading Spread in $',
'TickVolume', 'Net Asset Value Change%', 'ETF Change Price %',
'ETF Price'
] + etmoverslist]
ArbitrageDf = ArbitrageDf.round(5)
helperObj = Helper()
PriceDF['date'] = PriceDF['date'].apply(
lambda x: helperObj.getHumanTime(ts=x, divideby=1000) - timedelta(
hours=daylightSavingAdjutment))
ArbitrageDf['Timestamp'] = ArbitrageDf['Timestamp'].apply(
lambda x: str((helperObj.getHumanTime(ts=x, divideby=1000) -
timedelta(hours=daylightSavingAdjutment)).time()))
"""Add one minute to each to make end timestamp from start timestamp"""
PriceDF['date'] = PriceDF['date'] + timedelta(minutes=1)
ArbitrageDf['Timestamp'] = ArbitrageDf['Timestamp'].apply(
lambda x: str((datetime.strptime(x, '%H:%M:%S') + timedelta(
minutes=1)).time()))
ArbitrageDf.rename(columns={'Timestamp': 'Time'}, inplace=True)
res = {}
res['Prices'] = PriceDF[::-1]
# All day analysis
if callAllDayArbitrage:
PriceDFcopy = PriceDF.copy()
PriceDFcopy['date'] = PriceDFcopy['date'].apply(
lambda x: x.strftime('%H:%M:%S'))
PriceDFcopy.rename(columns={
'open': 'Open',
'close': 'Close',
'high': 'High',
'low': 'Low'
},
inplace=True)
cols_to_use_right = PriceDFcopy.columns.difference(
ArbitrageDf.columns)
ArbitrageDf = ArbitrageDf.merge(PriceDFcopy[cols_to_use_right],
left_on='Time',
right_on='date',
how='left')
arbitrageBuySellSignals, pnlstatementforday, scatterPlotData = calculateArbitrageResults(
df=ArbitrageDf,
etfname=etfname,
magnitudeOfArbitrageToFilterOn=0,
BuildMomentumSignals=False,
BuildPatternSignals=True,
includeMovers=True,
getScatterPlot=False)
cols_to_use = ArbitrageDf.columns.difference(
arbitrageBuySellSignals.columns)
arbitrageBuySellSignals['Time'] = arbitrageBuySellSignals.index
arbitrageBuySellSignals = arbitrageBuySellSignals.merge(
ArbitrageDf[cols_to_use],
left_on='Time',
right_on='Time',
how='left')
del arbitrageBuySellSignals['ETF Change Price %']
arbitrageBuySellSignals.rename(columns={'T': 'ETF Change Price %'},
inplace=True)
arbitrageBuySellSignals['ETFName'] = etfname
##arbitrageBuySellSignals['ETF Change Price %']=np.round(arbitrageBuySellSignals['ETF Change Price %'],2)
#arbitrageBuySellSignals['Price'] = np.round(arbitrageBuySellSignals['Price'],2)
#arbitrageBuySellSignals.Price=arbitrageBuySellSignals.Price.round(2)
arbitrageBuySellSignals[
'ETF Change Price %'] = arbitrageBuySellSignals[
'ETF Change Price %'].round(3)
arbitrageBuySellSignals = arbitrageBuySellSignals
res['Arbitrage'] = arbitrageBuySellSignals
res['pnlstatementforday'] = pnlstatementforday
else:
res['Arbitrage'] = ArbitrageDf
return res
except Exception as e:
print("Issue in Flask app while fetching ETF Description Data")
print(traceback.format_exc())
exc_type, exc_value, exc_tb = sys.exc_info()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=custom_server_logger)
def __init__(self):
self.helper_obj = Helper()
self.trade_prices_df_minutes = pd.DataFrame()
class ArbitrageCalculation():
def __init__(self):
self.helper_obj = Helper()
self.trade_prices_df_minutes = pd.DataFrame()
def fetch_all_data(self, etf_name, date):
"""Load the ETF Holding"""
etf_data = LoadHoldingsdata().LoadHoldingsAndClean(
etfname=etf_name, fundholdingsdate=date)
'''Load all the data - Holdings data for Etf, trade data, quotes data, open-close price'''
all_data = DataApi(etfname=etf_name, date=date, etfData=etf_data)
all_data.run_all_data_ops()
return etf_data, all_data
def trades_data_formatting_cleaning(self, all_data):
all_data.tradesDataDf['Time'] = all_data.tradesDataDf['Time'].apply(
lambda x: self.helper_obj.getHumanTime(ts=x, divideby=1000))
self.trade_prices_df_minutes = \
all_data.tradesDataDf.groupby([all_data.tradesDataDf['Time'], all_data.tradesDataDf['Symbol']])[
'Trade Price']
self.trade_prices_df_minutes = self.trade_prices_df_minutes.first(
).unstack(level=1)
price_for_nav_filling = all_data.openPriceData.set_index(
'Symbol').T.to_dict('records')[0]
market_start_time = datetime.datetime.strptime('13:29:00',
"%H:%M:%S").time()
market_end_time = datetime.datetime.strptime('20:00:00',
"%H:%M:%S").time()
mask = (self.trade_prices_df_minutes.index.time >=
market_start_time) & (self.trade_prices_df_minutes.index.time
<= market_end_time)
self.trade_prices_df_minutes = self.trade_prices_df_minutes[mask]
self.trade_prices_df_minutes = self.trade_prices_df_minutes.ffill(
axis=0)
self.trade_prices_df_minutes = self.trade_prices_df_minutes.fillna(
price_for_nav_filling)
return self.trade_prices_df_minutes
def calculate_etf_price_change(self, etf_name):
etf_price = self.trade_prices_df_minutes[etf_name]
self.trade_prices_df_minutes = self.trade_prices_df_minutes.pct_change(
).dropna() * 100
etf_price_change = self.trade_prices_df_minutes[etf_name]
return etf_price, etf_price_change
def calculate_etf_trading_spread(self, all_data):
"""Trading Spread Calculation"""
all_data.quotesDataDf['Time'] = all_data.quotesDataDf['Time'].apply(
lambda x: self.helper_obj.getHumanTime(ts=x, divideby=1000000000))
all_data.quotesDataDf['Time'] = all_data.quotesDataDf['Time'].map(
lambda x: x.replace(second=0))
all_data.quotesDataDf = all_data.quotesDataDf[
all_data.quotesDataDf['Bid Size'] != 0]
all_data.quotesDataDf = all_data.quotesDataDf[
all_data.quotesDataDf['Ask Size'] != 0]
all_data.quotesDataDf['Total Bid Ask Size'] = all_data.quotesDataDf[
'Ask Size'] + all_data.quotesDataDf['Bid Size']
all_data.quotesDataDf['Spread'] = all_data.quotesDataDf[
'Ask Price'] - all_data.quotesDataDf['Bid Price']
all_data.quotesDataDf['MidPrice'] = (
all_data.quotesDataDf['Ask Price'] +
all_data.quotesDataDf['Bid Price']) / 2
quotes_spreads_minutes = all_data.quotesDataDf.groupby(
'Time')['Spread'].mean()
return quotes_spreads_minutes
def calculateArbitrage(self, etf_name, date):
"""CALCULATE ETF ARBITRAGE FOR THE DATE"""
etf_data, all_data = self.fetch_all_data(etf_name, date)
'''Check if any holdings is trading in Non-US markets'''
if all_data.openPriceData is None:
return None
'''Trade Prices formatting between 13.29 and 20.00'''
self.trade_prices_df_minutes = self.trades_data_formatting_cleaning(
all_data)
'''Calculate Change in ETF Trade Price'''
etf_price, etf_price_change = self.calculate_etf_price_change(etf_name)
del self.trade_prices_df_minutes[etf_name]
'''Trading Spread Calculation'''
quotes_spreads_minutes = self.calculate_etf_trading_spread(all_data)
'''Arbitrage Calculation'''
net_asset_value_return = self.trade_prices_df_minutes.assign(
**etf_data.getETFWeights()).mul(
self.trade_prices_df_minutes).sum(axis=1)
ds = pd.concat([
etf_price, etf_price_change, net_asset_value_return,
quotes_spreads_minutes
],
axis=1).dropna()
ds.columns = [
'ETF Price', 'ETF Change Price %', 'Net Asset Value Change%',
'ETF Trading Spread in $'
]
ds['Arbitrage in $'] = (
ds['ETF Change Price %'] -
ds['Net Asset Value Change%']) * ds['ETF Price'] / 100
ds['Flag'] = 0
ds.loc[(abs(ds['Arbitrage in $']) > ds['ETF Trading Spread in $'])
& ds['ETF Trading Spread in $'] != 0, 'Flag'] = 111
ds['Flag'] = ds['Flag'] * np.sign(ds['Arbitrage in $'])
''' Movers and Changes '''
holdings_change = self.helper_obj.EtfMover(
df=self.trade_prices_df_minutes, columnName='Change%')
etf_mover_holdings = self.helper_obj.EtfMover(
df=self.trade_prices_df_minutes.assign(
**etf_data.getETFWeights()).mul(
self.trade_prices_df_minutes).dropna(axis=1),
columnName='ETFMover%')
'''Final Calculated Result DataFrame'''
ds = pd.concat([ds, etf_mover_holdings, holdings_change], axis=1)
print(ds)
return ds
class FetchPolygonData(object):
"""Fetch and Store Methods for Trades/Quotes Data from Polygon.io"""
def __init__(self,
date=None,
end_time='21:00:00',
end_time_loop='20:00:00',
polygon_method=None,
symbol_status=None,
collection_name=None,
insert_into_collection=None):
self.helperObj = Helper()
self.date = date
self.extract_data_till_time = self.helperObj.stringTimeToDatetime(
date=date, time=end_time_loop)
self.end_ts = self.helperObj.convertHumanTimeToUnixTimeStamp(
date=date, time=end_time)
self.polygon_method = polygon_method
self.insert_into_collection = insert_into_collection
self.collection_name = collection_name
self.symbol_status = symbol_status
def data_operation_runner(self, url):
"""Main for running Quotes data fetching operations"""
pagination = url
retry_counter = 2
while pagination:
response = self.response_from_api(pagination)
while retry_counter > 0 and response['success'] == False:
retry_counter -= 1
print("Response failure from polygon")
logger.error("Response failure from polygon")
response = self.response_from_api(pagination)
pagination_ = self.extract_data_from_response_and_store(response)
pagination = pagination_ if pagination_ != pagination else None
def response_from_api(self, url):
"""Get Quotes data from API URL"""
session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
response = session.get(url)
delay = response.headers.get("DELAY")
date = response.headers.get("DATE")
print("{}:{} with delay {}".format(date, response.url, delay))
json_data = json.loads(response.text)
return json_data
def extract_data_from_response_and_store(self, response):
"""Extract data from response, check for pagination, store the data"""
pagination_request = None
symbol = response['ticker']
print("Symbol being processed " + symbol)
response_data = [
dict(item, **{'Symbol': symbol}) for item in response['results']
if response['results']
]
last_unix_time_stamp = self.helperObj.getLastTimeStamp(response)
print("Fetched till time for =" +
str(self.helperObj.getHumanTime(last_unix_time_stamp)))
logger.debug("Fetched till time for = {}".format(
str(self.helperObj.getHumanTime(last_unix_time_stamp))))
if self.helperObj.checkTimeStampForPagination(
last_unix_time_stamp, self.extract_data_till_time):
pagination_request = self.polygon_method(
date=self.date,
symbol=symbol,
startTS=str(last_unix_time_stamp),
endTS=self.end_ts,
limitresult=str(50000))
self.symbol_status[symbol]['batchSize'] += 1
_ = self.insert_into_collection(
symbol=symbol,
datetosave=self.date,
savedata=response_data,
CollectionName=self.collection_name,
batchSize=self.symbol_status[symbol]['batchSize'])
print("Pagination Request = {}".format(pagination_request))
logger.debug("Pagination Request = {}".format(pagination_request))
if pagination_request:
return pagination_request
else:
print("No Pagination Required for = " + symbol)
logger.debug("No Pagination Required for = {}".format(symbol))
return None
"""Trades Get Operations"""
async def get_trade_data_and_save(self, url, session):
"""Get Trades data from API URL Asynchronously"""
try:
async with session.get(url) as response:
delay = response.headers.get("DELAY")
date = response.headers.get("DATE")
print("{}:{} with delay {}".format(date, response.url, delay))
json_data = json.loads(await response.text())
symbol = json_data['ticker']
if 'results' not in json_data:
print(f"No results for {symbol}")
data = [
dict(item, **{'Symbol': symbol})
for item in json_data['results'] if 'results' in json_data
]
self.insert_into_collection(
symbol=symbol,
datetosave=self.date,
savedata=data,
CollectionName=self.collection_name)
print(f'inserted {symbol} data')
except KeyError:
print(
f"################################ No results for {symbol} ################################"
)
logger.warn(
f"################################ No results for {symbol} ################################"
)
async def task_creator_and_gatherer(self, func, iterable):
"""Asynchronous Main for Trades data fetching operations"""
async with ClientSession() as session:
tasks = [
asyncio.ensure_future(func(url, session)) for url in iterable
]
return await asyncio.gather(*tasks)