def run(self):
"""Main function to run Daily Open Close Operation"""
try:
symbols_to_be_downloaded = self.leave_existing_and_find_symbols_to_be_downloaded(
symbols=self.symbols,
date=self.date,
CollectionName=self.collection_name)
combined_data = []
if len(symbols_to_be_downloaded) > 0:
create_new_urls = self.create_urls(
symbolsToBeDownloaded=symbols_to_be_downloaded)
failed_tickers = self.get_save_open_close_data(
openCloseURLs=create_new_urls)
if failed_tickers:
if getpass.getuser() == 'ubuntu':
conn = MongoDBConnectors(
).get_pymongo_readonly_production_production()
else:
conn = MongoDBConnectors(
).get_pymongo_readonly_devlocal_production()
conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
pipeline = return_daily_open_close_pipeline(
failed_tickers, self.date)
data_cursor = conn.ETF_db.DailyOpenCloseCollection.aggregate(
pipeline)
combined_data = [data for data in data_cursor]
data = self.fetch_data()
data.extend(combined_data)
return pd.DataFrame(data)
except Exception as e:
logger.exception(e)
logger2.exception(e)
def __init__(self):
self.etflist = pd.read_csv('../CSVFiles/250M_WorkingETFs.csv').columns.to_list()
if getpass.getuser() == 'ubuntu':
self.connection = MongoDBConnectors().get_pymongo_readonly_production_production()
else:
self.connection = MongoDBConnectors().get_pymongo_readonly_devlocal_production()
self.connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
self.db = self.connection.ETF_db
pass
def __init__(self):
self.system_username = getpass.getuser()
if self.system_username == 'ubuntu':
''' Production to Production readWrite '''
self.conn = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
''' Dev Local to Production Read Only '''
# self.conn = MongoDBConnectors().get_pymongo_readonly_devlocal_production()
''' Dev Local to Dev Local readWrite '''
self.conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
def __init__(self):
self.sysUserName = getpass.getuser()
if self.sysUserName == 'ubuntu':
self.connforthis = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
self.connforthis = MongoDBConnectors(
).get_pymongo_readonly_devlocal_production()
self.connforthis = MongoDBConnectors().get_pymongo_devlocal_devlocal()
self.arbitragecollection = self.connforthis.ETF_db.ArbitrageCollectionNew
def collections_dropper():
sys_private_ip = socket.gethostbyname(socket.gethostname())
if sys_private_ip == '172.31.76.32':
connection = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
per_sec_trades_db = connection.ETF_db.PerSecLiveTrades
per_sec_quotes_db = connection.ETF_db.PerSecLiveQuotes
connection.ETF_db.drop_collection(per_sec_trades_db)
connection.ETF_db.drop_collection(per_sec_quotes_db)
def __init__(self):
self.cashvalueweight = None
self.weights = None
self.symbols = None
self.system_username = getpass.getuser()
sys_private_ip = socket.gethostbyname(socket.gethostname())
if sys_private_ip == '172.31.76.32' and self.system_username == 'ubuntu':
self.conn = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
# self.conn = MongoDBConnectors().get_pymongo_readonly_devlocal_production()
self.conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
def __init__(self, etf_name=None, date_=None):
sys_private_ip = socket.gethostbyname(socket.gethostname())
if sys_private_ip == '172.31.76.32':
self.connection = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
self.connection = MongoDBConnectors(
).get_pymongo_devlocal_devlocal()
self.per_sec_live_trades = self.connection.ETF_db.PerSecLiveTrades
self.per_sec_live_trades.create_index([("Symbol", ASCENDING),
("t", DESCENDING)])
self.per_sec_live_quotes = self.connection.ETF_db.PerSecLiveQuotes
self.per_sec_live_quotes.create_index([("Symbol", ASCENDING),
("t", DESCENDING)])
self.date = date_
self.etf_name = etf_name
def Save_PnLData(self, data):
if self.sysUserName == 'ubuntu':
result = self.connforthis.ETF_db.PNLDataCollection.insert_many(
data)
else:
result = MongoDBConnectors().get_pymongo_devlocal_devlocal(
).ETF_db.PNLDataCollection.insert_many(data)
print('inserted %d docs' % (len(result.inserted_ids), ))
logger.debug('inserted %d docs' % (len(result.inserted_ids), ))
def remove_old_trades_quotes_for_the_date_etf(self):
"""REMOVE ALL QUOTES DATA AND TRADES DATA FOR THE UPDATED ETF LIST FOR THE GIVEN DATE"""
try:
del_list = self.etflist.copy()
if getpass.getuser() == 'ubuntu':
rem_conn = MongoDBConnectors().get_pymongo_readWrite_production_production()
else:
rem_conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
quotes_del = rem_conn.ETF_db.QuotesData.delete_many(
{'dateForData': datetime.strptime(self.date, '%Y-%m-%d'), 'symbol': {'$in': del_list}})
print(quotes_del.deleted_count)
sym_list = [del_list.extend(
LoadHoldingsdata().LoadHoldingsAndClean(etf, datetime.strptime(self.date, '%Y-%m-%d')).getSymbols()) for
etf
in self.etflist]
trades_del = rem_conn.ETF_db.TradesData.delete_many(
{'dateForData': datetime.strptime(self.date, '%Y-%m-%d'), 'symbol': {'$in': del_list}})
print(trades_del.deleted_count)
except Exception as e:
logger.exception(e)
logger2.exception(e)
pass
sys.path.append("../")
import pandas as pd
import numpy as np
from dateutil import tz
from datetime import datetime, timedelta, date
from MongoDB.MongoDBConnections import MongoDBConnectors
from CalculateETFArbitrage.Helpers.LoadEtfHoldings import LoadHoldingsdata
from CommonServices.Holidays import LastWorkingDay
from PolygonTickData.Helper import Helper
from dataclasses import dataclass
sys_private_ip = socket.gethostbyname(socket.gethostname())
if sys_private_ip == '172.31.76.32':
connection = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
per_sec_trades_db = connection.ETF_db.PerSecLiveTrades
per_sec_quotes_db = connection.ETF_db.PerSecLiveQuotes
helper_object = Helper()
class TradeStruct(
): # For Trade Objects, containing current minute and last minute price for Tickers
__slots__ = ['symbol', 'priceT', 'priceT_1', 'price_pct_chg']
def calc_pct_chg(self, priceT, priceT_1):
if priceT_1 == 0:
return 0
return ((priceT - priceT_1) / priceT_1) * 100
fetchETFsWithSimilarTotAsstUndMgmt, fetchOHLCHistoricalData
from CalculateETFArbitrage.Helpers.LoadEtfHoldings import LoadHoldingsdata
from FlaskAPI.Components.ETFArbitrage.ETFArbitrageMain import RetrieveETFArbitrageData, retrievePNLForAllDays, \
OverBoughtBalancedOverSold, CandlesignalsColumns
from FlaskAPI.Components.ETFArbitrage.helperForETFArbitrage import etfMoversChangers
from MongoDB.PerMinDataOperations import PerMinDataOperations
from FlaskAPI.Components.LiveCalculations.helperLiveArbitrageSingleETF import fecthArbitrageANDLivePrices, \
analyzeSignalPerformane, CategorizeSignals
from CommonServices.Holidays import LastWorkingDay, HolidayCheck
from FlaskAPI.Helpers.ServerLogHelper import custom_server_logger
from functools import lru_cache
server_logger = custom_server_logger
# connection = MongoDBConnectors().get_pymongo_readonly_devlocal_production()
connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
app = flaskAppMaker().create_app()
CORS(app)
api_auth_object = authAPI()
@app.before_request
def before_request():
g.start = time.time()
# if sys.platform.startswith('linux') and getpass.getuser() == 'ubuntu':
# flaskAppMaker().get_index_page()
import datetime
from pymongo import ASCENDING, DESCENDING
import getpass, socket
from MongoDB.MongoDBConnections import MongoDBConnectors
system_username = getpass.getuser()
system_private_ip = socket.gethostbyname(socket.gethostname())
if system_username == 'ubuntu' and system_private_ip == '172.31.76.32':
''' Production to Production:'''
connection = MongoDBConnectors(
).get_pymongo_readWrite_production_production()
else:
''' Dev Local to Production ReadOnly:
Will need to comment out create_index() statements to use this connection '''
# connection = MongoDBConnectors().get_pymongo_readonly_devlocal_production()
''' Dev Local to Production ReadWrite:
Will need to specify username password file in MongoDB/MongoDBConnections.py __init__() '''
# connection = MongoDBConnectors().get_pymongo_readWrite_devlocal_production()
''' Dev Local to Dev Local '''
connection = MongoDBConnectors().get_pymongo_devlocal_devlocal()
db = connection.ETF_db
if system_username == 'ubuntu' and system_private_ip == '172.31.76.32':
''' Production to Production:'''
motor_client = MongoDBConnectors(
).get_motorasync_readWrite_production_production()
else:
''' Dev Local to Production ReadOnly:
Will need to comment out create_index() statements to use this connection '''
def retrievePNLForAllDays(self, magnitudeOfArbitrageToFilterOn=0):
try:
date_list = self.returnres()
date_list.sort()
date_list = [
date for date in date_list
if date > datetime.datetime(2020, 6, 4)
]
etflist = pd.read_csv(
'../../CSVFiles/250M_WorkingETFs.csv').columns.to_list()
# final_res = []
for date in date_list:
print(date)
try:
if self.sysUserName == 'ubuntu':
presence = MongoDBConnectors(
).get_pymongo_readWrite_production_production(
).ETF_db.PNLDataCollection.find({'Date': date})
else:
presence = MongoDBConnectors(
).get_pymongo_devlocal_devlocal(
).ETF_db.PNLDataCollection.find({'Date': date})
presence_list = list(presence)
etf_already_present = [
item['Symbol'] for item in presence_list
]
etf_to_be_done = list(
set(etflist) - set(etf_already_present))
if len(presence_list) == len(etflist):
continue
all_etf_arb_cursor = self.arbitragecollection.find(
{'dateOfAnalysis': date})
all_etf_arb = list(all_etf_arb_cursor)
hist_arb_present_syms = list(
set([etf['ETFName'] for etf in all_etf_arb]))
if len(hist_arb_present_syms) == len(presence_list):
print('ETFs with Hist Arb present already done')
continue
PNLOverDates = {}
final_res = []
all_etf_arb = [[
etf_arb for etf_arb in all_etf_arb
if etf_arb['ETFName'] == etf
] for etf in etf_to_be_done]
# Iter over the collection results
for etf_arblist in all_etf_arb:
# if etf_arb['ETFName'] in etf_to_be_done:
if len(etf_arblist) < 1:
continue
try:
print(etf_arblist[0]['ETFName'])
logger.debug(etf_arblist[0]['ETFName'])
allData, pricedf, pnlstatementforday, scatterPlotData = AnalyzeArbitrageDataForETF(
arbitrageDataFromMongo=etf_arblist,
magnitudeOfArbitrageToFilterOn=
magnitudeOfArbitrageToFilterOn)
PNLOverDates[str(etf_arblist[0]
['ETFName'])] = pnlstatementforday
except Exception as e0:
print("Exception in {}".format(
etf_arblist[0]['ETFName']))
logger.warning("Exception in {}".format(
etf_arblist[0]['ETFName']))
print(e0)
traceback.print_exc()
logger.exception(e0)
pass
PNLOverDates = pd.DataFrame(PNLOverDates).T
# del PNLOverDates['Magnitue Of Arbitrage']
PNLOverDates.rename(columns={
'PNL% Sell Pos. (T+1)': 'Sell Return%',
'PNL% Buy Pos. (T+1)': 'Buy Return%',
'Magnitue Of Arbitrage': 'Magnitue Of Arbitrage',
'# of Buy Signal': '# T_Buy',
'# of Right Buy Signal': '# R_Buy',
'# of Sell Signal': '# T_Sell',
'# of Right Sell Signal': '# R_Sell'
},
inplace=True)
PNLOverDates['% R_Buy'] = round(
PNLOverDates['# R_Buy'] / PNLOverDates['# T_Buy'], 2)
PNLOverDates['% R_Sell'] = round(
PNLOverDates['# R_Sell'] / PNLOverDates['# T_Sell'], 2)
PNLOverDates['Date'] = date
PNLOverDates = PNLOverDates[[
'Date', 'Sell Return%', 'Buy Return%', '# T_Buy',
'# R_Buy', '% R_Buy', '# T_Sell', '# R_Sell',
'% R_Sell', 'Magnitue Of Arbitrage'
]]
final_res.extend(
PNLOverDates.reset_index().rename(columns={
'index': 'Symbol'
}).to_dict('records'))
print(final_res)
self.Save_PnLData(final_res)
except Exception as e:
traceback.print_exc()
logger.exception(e)
# print("Exception in {}".format(etf_arb['ETFName']))
pass
except Exception as e1:
logger.exception(e1)
traceback.print_exc()
pass
