def get_and_save_quotes_data(etfname, date):
etfname = etfname
date = date
etfData = LoadHoldingsdata().LoadHoldingsAndClean(etfname=etfname,
fundholdingsdate=date)
ob = TradesQuotesProcesses(symbols=etfData.getSymbols(), date=date)
ob.fetch_and_store_runner(collection_name=quotesCollection)
quotesDataDf = ob.get_data(collection_name=quotesCollection,
pipeline=quotespipeline)
print(quotesDataDf)
print("SAVED Quotes for {} {}".format(etfname, date))
def get_unique_ticker_list_for_trades(top_10_etf_list):
load_holdings_object = LoadHoldingsdata()
unique_ticker_list = [
load_holdings_object.LoadHoldingsAndClean(
etfname=etf, fundholdingsdate=_date).getSymbols()
for etf in top_10_etf_list
]
unique_ticker_list = list(chain.from_iterable(unique_ticker_list))
unique_ticker_list = [
ticker.split(' ')[0] for ticker in unique_ticker_list
]
unique_ticker_list = list(set(unique_ticker_list))
return unique_ticker_list
def all_process_runner_trades(self, symbols=None, date_for=None):
if not symbols:
etf_data = LoadHoldingsdata().LoadHoldingsAndClean(
etfname=self.etf_name, fundholdingsdate=self.date)
symbols = etf_data.getSymbols()
# symbols_to_download = [symbol for symbol in symbols if
# not self.symbol_check_trades(symbol=symbol, collection=self.per_sec_live_trades)]
trades_quotes_proc_obj = TradesQuotesProcesses(symbols=symbols,
date=date_for)
print("Processing historic trade data")
trades_quotes_proc_obj.trades_fetch_and_store_runner_live(
collection_name=self.connection.ETF_db.PerSecLiveTrades,
per_sec_create_url_func=self.create_urls_for_trades)
def getAndSaveTradesData(etfname, date):
etfname = etfname
date = date
etfData = LoadHoldingsdata().LoadHoldingsAndClean(etfname=etfname,
fundholdingsdate=date)
ob = TradesQuotesProcesses(symbols=etfData.getSymbols(), date=date)
ob.fetch_and_store_runner(collection_name=tradeCollection,
trade_data_flag=True)
tradesDataDf = ob.get_data(collection_name=tradeCollection,
pipeline=tradespipeline)
tradesDataDf['Trade Price'] = (tradesDataDf['High Price'] +
tradesDataDf['Low Price']) / 2
print(tradesDataDf)
print("SAVED {} {}".format(etfname, date))
def fetch_all_data(self, etf_name, date):
"""Load the ETF Holding"""
etf_data = LoadHoldingsdata().LoadHoldingsAndClean(
etfname=etf_name, fundholdingsdate=date)
'''Load all the data - Holdings data for Etf, trade data, quotes data, open-close price'''
all_data = DataApi(etfname=etf_name, date=date, etfData=etf_data)
all_data.run_all_data_ops()
return etf_data, all_data
def ETFandHoldingsData(ETFName, date):
try:
MongoDBConnectors().get_mongoengine_devlocal_devlocal()
# Load all the data holdings data together
etfdata = LoadHoldingsdata().getAllETFData(ETFName, date)
if type(etfdata) == Response:
return etfdata
ETFDataObject = etfdata.to_mongo().to_dict()
print(ETFDataObject)
HoldingsDatObject = pd.DataFrame(ETFDataObject['holdings']).set_index('TickerSymbol').T.to_dict()
ETFDataObject['TotalAssetsUnderMgmt'] = "${:,.3f} M".format(ETFDataObject['TotalAssetsUnderMgmt'] / 1000)
ETFDataObject['SharesOutstanding'] = "{:,.0f}".format(ETFDataObject['SharesOutstanding'])
ETFDataObject['InceptionDate'] = str(ETFDataObject['InceptionDate'])
# List of columns we don't need
for v in ['_id', 'DateOfScraping', 'ETFhomepage', 'holdings', 'FundHoldingsDate']:
del ETFDataObject[v]
ETFDataObject = pd.DataFrame(ETFDataObject, index=[0])
ETFDataObject = ETFDataObject.replace(np.nan, 'nan', regex=True)
ETFDataObject = ETFDataObject.loc[0].to_dict()
allData = {}
allData['ETFDataObject'] = ETFDataObject
allData['HoldingsDatObject'] = HoldingsDatObject
print(ETFDataObject)
print(allData['HoldingsDatObject'])
return allData
except Exception as e:
print("Issue in Flask app while fetching ETF Description Data")
print(traceback.format_exc())
return str(e)
def fetchHoldingsData(ETFName, StartDate):
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
try:
etfdata = LoadHoldingsdata().getAllETFData(ETFName, StartDate)
if type(etfdata) == Response:
return etfdata
ETFDataObject = list(etfdata)[0]
# HoldingsDatObject=pd.DataFrame(ETFDataObject['holdings']).set_index('TickerSymbol').round(2).T.to_dict()
# print(HoldingsDatObject)
return jsonify(ETFDataObject['holdings'])
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def ETFHoldJsonData(self, etfname):
try:
df = None
df = LoadHoldingsdata().getHoldingsDatafromDB(
etfname, (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime("%Y-%m-%d"))
return self.convertDataToDict(df, etfname)
except Exception as e:
self.raiseError(errorType=1)
# Get the last date from the mongodb this time usine Schemas.py Pymongo, we are fetching last occurence
etfdata = etfholdings_collection.find({
'ETFTicker': etfname
}).sort("FundHoldingsDate", -1).limit(1)
for data in etfdata:
df = pd.DataFrame(data['holdings'])
return self.convertDataToDict(
df, etfname) if df is not None else self.raiseError(
errorType=2)
def getDailyChangeUnderlyingStocks(ETFName, date):
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
if checkifDateIsBeforeJuneFive(date):
return CustomAPIErrorHandler().handle_error(
'Data only available before June 5th 2020, please choose a date after 5th June',
500)
try:
etfdata = LoadHoldingsdata().getAllETFData(ETFName, date)
if type(etfdata) == Response:
return etfdata
ETFDataObject = list(etfdata)[0]
TickerSymbol = pd.DataFrame(
ETFDataObject['holdings'])['TickerSymbol'].to_list()
TickerSymbol.remove('CASH') if 'CASH' in TickerSymbol else TickerSymbol
openclosedata_cursor = connection.ETF_db.DailyOpenCloseCollection.find(
{
'dateForData': datetime.strptime(date, '%Y%m%d'),
'Symbol': {
'$in': TickerSymbol
}
}, {'_id': 0})
responses = list(openclosedata_cursor)
responses = pd.DataFrame.from_records(responses)
responses['DailyChangepct'] = (
(responses['Close'] - responses['Open Price']) /
responses['Open Price']) * 100
responses['DailyChangepct'] = responses['DailyChangepct'].round(3)
responses.rename(columns={
'Symbol': 'symbol',
'Volume': 'volume'
},
inplace=True)
return jsonify(responses[['symbol', 'DailyChangepct',
'volume']].to_dict(orient='records'))
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def remove_old_trades_quotes_for_the_date_etf(self):
"""REMOVE ALL QUOTES DATA AND TRADES DATA FOR THE UPDATED ETF LIST FOR THE GIVEN DATE"""
try:
del_list = self.etflist.copy()
if getpass.getuser() == 'ubuntu':
rem_conn = MongoDBConnectors().get_pymongo_readWrite_production_production()
else:
rem_conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
quotes_del = rem_conn.ETF_db.QuotesData.delete_many(
{'dateForData': datetime.strptime(self.date, '%Y-%m-%d'), 'symbol': {'$in': del_list}})
print(quotes_del.deleted_count)
sym_list = [del_list.extend(
LoadHoldingsdata().LoadHoldingsAndClean(etf, datetime.strptime(self.date, '%Y-%m-%d')).getSymbols()) for
etf
in self.etflist]
trades_del = rem_conn.ETF_db.TradesData.delete_many(
{'dateForData': datetime.strptime(self.date, '%Y-%m-%d'), 'symbol': {'$in': del_list}})
print(trades_del.deleted_count)
except Exception as e:
logger.exception(e)
logger2.exception(e)
pass
def SendETFHoldingsData(ETFName, date):
res = api_auth_object.authenticate_api()
if type(res) == Response:
return res
try:
allData = {}
etfdata = LoadHoldingsdata().getAllETFData(ETFName, date)
if type(etfdata) == Response:
return etfdata
ETFDataObject = list(etfdata)[0]
allData['SimilarTotalAsstUndMgmt'] = fetchETFsWithSimilarTotAsstUndMgmt(
connection=connection,
totalassetUnderManagement=ETFDataObject['TotalAssetsUnderMgmt'])
ETFDataObject['TotalAssetsUnderMgmt'] = "${:,.3f} M".format(
ETFDataObject['TotalAssetsUnderMgmt'] / 1000)
ETFDataObject['SharesOutstanding'] = "{:,.0f}".format(
ETFDataObject['SharesOutstanding'])
ETFDataObject['InceptionDate'] = str(ETFDataObject['InceptionDate'])
# List of columns we don't need
for v in [
'_id', 'DateOfScraping', 'ETFhomepage', 'holdings',
'FundHoldingsDate'
]:
del ETFDataObject[v]
ETFDataObject = pd.DataFrame(ETFDataObject, index=[0])
ETFDataObject = ETFDataObject.replace(np.nan, 'nan', regex=True)
ETFDataObject = ETFDataObject.loc[0].to_dict()
allData['ETFDataObject'] = ETFDataObject
return json.dumps(allData)
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exc()
return MultipleExceptionHandler().handle_exception(
exception_type=exc_type, e=e, custom_logger=server_logger)
def make_etf_objects(_date, etf_name):
holdings_dict = LoadHoldingsdata().LoadHoldingsAndClean(
etfname=etf_name, fundholdingsdate=_date).getETFWeights()
return ETF(etf_name=etf_name, holdings_dict=holdings_dict)
def get_ticker_list_for_etf(date, etf_name) -> list:
ticker_list = LoadHoldingsdata().LoadHoldingsAndClean(
etfname=etf_name, fundholdingsdate=date).getSymbols()
return ticker_list