def getHoldings(investType, filename, month, year):
# Get symbols of interest & sort -> Update Fund vs ETF
if (investType.lower().strip() == 'fund'):
symbols = readFunds('Symbols.csv')
#symbols = readFunds('SymbolsDebug.csv')
elif (investType.lower().strip() == 'etf'):
symbols = readFunds('SymbolsETF.csv')
#symbols = readFunds('SymbolsETFDebug.csv')
else:
print('Type should be fund or etf')
return
sortSymbols(symbols) # Sort symbols & remove duplicates
# String to locate fund holdings
URL = 'https://www.marketwatch.com/investing/fund/'
URLLong = '/holdings'
# prealloc container
holdingsDict = {}
symbolsData = []
# top of the spreadsheet
symbolsData.append(
['Fund', 'Company', 'Symbol', 'Total Net Assets', 'Monthly Rtn'])
# Build the list of holdings for each fund
for symbol in symbols.index:
print(symbol)
URLSym = URL + symbol.strip()
#URLSym += URLLong #Get top 25 instead of top 10
# Lookup the holdings
pageProc = procRequest(URLSym)
holdings = " "
holdings, hDictSub, asOf, closed = buildHoldings(pageProc, symbol)
if (closed == True):
#print('closed')
strList = [symbol] + ['closed']
else:
holdingsDict.update(hDictSub) #Merge dictionaries - one lookup
strList = [symbol] + [asOf]
# Build table to serialize, note mergeDict will add value info
symbolsData.append(strList)
for i in holdings:
symbolsData.append([" "] + i)
#Lookup holdings
for symbol in holdingsDict:
holdingsDict[symbol] = lookupSymbol(symbol, month, year)
print(symbol, "\t->\t", holdingsDict[symbol])
# Add value info into the symbolsData
mergeDictList(holdingsDict, symbolsData) # merge symbol returns into list
seralizeData(filename, symbolsData)
def getETFListings(letter, URL, fundList):
mwPage = 'init'
numCount = 1
while mwPage is not None:
# Website splits table up over many pages...
URLSym = URL + letter + '/' + str(numCount)
mwPage = procRequest(URLSym, 1,
False) # need to append a letter 'A'->'Z'
mwPage = mwReducePage(mwPage)
if mwPage is None:
break
mwBuildFunds(mwPage, fundList)
numCount += 1
return fundList
def quartelyFundMetrics(filename):
symbols = readFunds('Symbols.csv') #Get symbols of interest
#symbols = readFunds('SymbolsDebug.csv')
# Sort symbols & remove duplicates
sortSymbols(symbols)
# String to locate fund holdings
URL = 'https://www.marketwatch.com/investing/fund/'
dataList = [] # Allocate list to hold data
hdrList = [] # Allocate list to hold header
dateToday = date.today().strftime('%Y-%m-%d')
count = True # Build header
for symbol in symbols.index: # lookup data
print(symbol)
URLSym = URL + symbol
try:
yfData, yfDataHdr = yfGetQuarterlyMetrics(symbol)
mwPage = procRequest(URLSym)
mwKeyDataList, mwKeyDataHdr = mwGetKeyData(mwPage)
mwPerfomanceList, mwPerformanceHdr = mwGetReturns(mwPage)
mwRiskList, mwRiskHdr = mwGetRiskDetails(mwPage)
mwFundList, mwFundHdr = mwGetFundDetails(mwPage)
symList = list(
itertools.chain([dateToday], yfData, mwKeyDataList,
mwPerfomanceList, mwRiskList, mwFundList))
dataList.append(symList)
if (count == True):
count = False
hdrList = list(
itertools.chain(['Date'], yfDataHdr, mwKeyDataHdr,
mwPerformanceHdr, mwRiskHdr, mwFundHdr))
except (IndexError, KeyError):
dataList.append([dateToday, symbol])
#Seralize data
seralizeData(filename, dataList, hdrList) # Seralize data
def allMutualFunds(filename):
# String to locate funds
URL = 'https://www.marketwatch.com/tools/mutual-fund/list/'
fundList = [] # Allocate list to hold data
ltrCount = 1
for letter in ascii_lowercase:
drawProgressBar(ltrCount / len(ascii_lowercase))
ltrCount += 1
URLSym = URL + letter
mwPage = procRequest(URLSym, 1,
False) # need to append a letter 'A'->'Z'
mwPage = mwReducePage(mwPage)
mwBuildFunds(mwPage, fundList)
print('')
cols = ['Symbol', 'Description']
seralizeData(filename, fundList, cols) # Seralize data
def quarterlyETFMetric(filename):
symbols = readFunds('SymbolsETF.csv') #Get symbols of interest
#symbols = readFunds('SymbolsETFDebug.csv')
# Sort symbols & remove duplicates
sortSymbols(symbols)
# String to locate fund holdings
MW_URL = 'https://www.marketwatch.com/investing/fund/'
YF_URL_p1 = 'https://finance.yahoo.com/quote/'
YF_URL_p2 = '/performance'
YF_URL_p3 = '/profile'
dataList = [] # Allocate list to hold data
hdrList = [] # Allocate list to hold header
dateToday = date.today().strftime('%Y-%m-%d')
count = True # Build header
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
}
for symbol in symbols.index: # lookup data
print(symbol)
MW_URLSym = MW_URL + symbol.strip()
YF1_URLSym = YF_URL_p1 + symbol.strip() + YF_URL_p2
YF2_URLSym = YF_URL_p1 + symbol.strip() + YF_URL_p3
# Prealloc storage variables
mwKeyDataList = ''
mwKeyDataHdr = ''
yfPerfData = ''
yfPerfDataHdr = ''
yfProData = ''
yfProDataHdr = ''
try:
mwPage = procRequest(MW_URLSym)
nameLong, closed = mwGetName(mwPage)
if (closed is False):
mwKeyDataList, mwKeyDataHdr = mwGetKeyData(mwPage)
yfPage = procRequest(YF1_URLSym, 5, False, headers=headers)
yfPerfData, yfPerfDataHdr = yfPerformance(yfPage)
yfPage = procRequest(YF2_URLSym, 5, False, headers=headers)
yfProData, yfProDataHdr = yfProfile(yfPage)
symList = list(
itertools.chain([dateToday], [symbol], [nameLong],
mwKeyDataList, yfPerfData, yfProData))
dataList.append(symList)
else:
dataList.append([dateToday, symbol, 'closed'])
if (count == True):
count = False
hdrList = list(
itertools.chain(['Date'], ['Symbol'], ['Long Name'],
mwKeyDataHdr, yfPerfDataHdr, yfProDataHdr))
except (IndexError, KeyError):
dataList.append([dateToday, symbol])
#Seralize data
seralizeData(filename, dataList, hdrList) # Seralize data