def insertSymbolinDatabase(symbol):
index = load('data/index.p')
try:
print index[symbol]
print symbol + 'is already in our database'
except:
result = searchSymbol(symbol)
if result == []:
print 'symbol not found'
return
nse,bse = getSymbolFromMCurl(result[-1])
symbols = load('data/symbols.p')
data = load('data/symbolsMCupdated.p')
l = len(data)
symbols.append(['',nse,bse])
data.append(['',nse,bse])
data[-1].extend(result)
if nse != "":
index[nse] = l
if bse != "":
index[bse] = l
dumpData(index, 'data/index.p')
dumpData(symbols, 'data/symbols.p')
dumpData(data, 'data/symbolsMCupdated.p')
return True
def insertSymbolinDatabase(symbol):
index = load('data/index.p')
try:
print index[symbol]
print symbol + 'is already in our database'
except:
result = searchSymbol(symbol)
if result == []:
print 'symbol not found'
return
nse, bse = getSymbolFromMCurl(result[-1])
symbols = load('data/symbols.p')
data = load('data/symbolsMCupdated.p')
l = len(data)
symbols.append(['', nse, bse])
data.append(['', nse, bse])
data[-1].extend(result)
if nse != "":
index[nse] = l
if bse != "":
index[bse] = l
dumpData(index, 'data/index.p')
dumpData(symbols, 'data/symbols.p')
dumpData(data, 'data/symbolsMCupdated.p')
return True
def updateDictsWithDEratio(a, b):
stockSymbols = load('data/symbolsMCupdated.p')
for index in range(a, b):
print index, 7630, stockSymbols[index][1], stockSymbols[index][2]
DIR = 'data/financials/' + str(index) + '/'
annual = load(DIR + 'annualDict.p')
if annual is not None:
annual['DEratio'] = getDebtList(index)
dumpData(annual, DIR + 'annualDict.p')
def saveQuarterDict(index):
DIR = 'data/financials/' + str(index) + '/'
quarterData1 = load(DIR + 'quarterFinancialData1.p')
DictData1 = processAndDump(quarterData1, DIR + 'quarterData1.txt')
if DictData1 is None:
return
quarterData2 = load(DIR + 'quarterFinancialData2.p')
DictData2 = processAndDump(quarterData2, DIR + 'quarterData2.txt')
DictData = combine(DictData1, DictData2)
if DictData is not None:
DictData['index'] = index
dumpData(DictData, DIR + 'quarterDict.p')
def getParsedFinancialData(self, index):
'''DIR = 'data/financials/'+str(index)+'/'
annualData = load(DIR+'annualDict.p')
if annualData == None:
return {}
output = {}
output['Year'] = annualData['month']
output['Annual EPS'] = annualData['EPS Before Extra Ordinary']['Basic EPS']
output['Net Annual Sales'] = annualData['Net Sales/Income from operations']
'''
try:
DIR = 'data/financials/' + str(index) + '/'
return ((load(DIR + 'annualDict.p'), load(DIR + 'quarterDict.p')),
0)
except:
return (None, 0)
def __init__(self):
self.completeFinancialData = {}
self.filteredFinancialData = {}
self.stockSymbols = load('data/symbolsMCupdated.p')
self.indexes = load('data/index.p')
self.nSymbols = len(self.stockSymbols)
self.epsQuarterThreshold = 20
self.epsAnnualThreshold = 20
self.salesThreshold = 25
self.filters = [
self.epsQuarterFilter, self.salesQuarterFilter, self.ATPMfilter,
self.epsAnnualFilter, self.eps2QuarterFilter, self.ROEFilter,
self.DEFilter
]
self.filterCount = len(self.filters)
self.excludeSymbolList = load('data/excludeSymbols.p')
def saveAnnualDict(index):
DIR = 'data/financials/' + str(index) + '/'
annualData = load(DIR + 'annualFinancialData.p')
DictData = processAndDump(annualData, DIR + 'annualData.txt')
if DictData is not None:
DictData['index'] = index
dumpData(DictData, DIR + 'annualDict.p')
def updateDicts(a, b):
for index in range(a, b):
print index, 7630
DIR = 'data/financials/' + str(index) + '/'
try:
annual = load(DIR + 'annualDict.p')
if annual is not None:
annual['index'] = index
dumpData(annual, DIR + 'annualDict.p')
except:
pass
try:
quarter = load(DIR + 'quarterDict.p')
if quarter is not None:
quarter['index'] = index
dumpData(quarter, DIR + 'quarterDict.p')
except:
pass
def getDebtList(index):
DIR = 'data/financials/' + str(index) + '/'
DEratioData = load(DIR + 'ratios.p')
parsedData = getParsedSoupFromHTML(DEratioData)
tables = parsedData.findAll('table')
l = len(tables)
for i in range(l - 1, -1, -1):
data = getText(tables[i]).strip('\n')
if data == '':
continue
lines = data.split('\n')
if lines[0].startswith('Data Not Available'):
return None
else:
try:
month[lines[0][:3]]
except:
continue
#try:
if month[lines[0][:3]] == 1:
j = 0
lines = map(stripNewLine, lines)
try:
while True:
mon = lines[j].split(' ')[0]
if month[mon] == 1:
j = j + 1
except:
if j == 0:
return None
months = j
while j < len(lines):
if lines[j] == 'Debt Equity Ratio':
DEratio = []
if lines[j + 1] == "":
j = j + 1
for k in range(j + 1, min(j + 1 + 3, j + 1 + months)):
if lines[k] == '--':
return DEratio
DEratio.append(float(lines[k]))
return DEratio
j = j + 1
#except:
# continue
return None
from commonSettings import *
from commonFunctions import getParsedHTML, getText, dumpData, csvWriter, load, getHTML
StockSymbols = load('data/symbolsMCupdated.p')
Indexes = load('data/index.p')
nSymbols = len(StockSymbols)
def saveFinancialData(data, index):
directory = 'data/financials/'+str(index)+'/'
if not os.path.exists(directory):
os.makedirs(directory)
dumpData(data[0], directory+'annualFinancialData.p')
dumpData(data[1], directory+'quarterFinancialData1.p')
dumpData(data[2], directory+'quarterFinancialData2.p')
def download_and_save_fin_data_from_symbol(symbol):
try:
index = Indexes[symbol]
url = StockSymbols[index][-1]
data = download_fin_data_from_URL(url)
saveFinancialData(data, index)
except:
print 'Symbol not found'
def download_fin_data_from_URL(url, driver = ""):
if url.startswith('http://www.moneycontrol.com/india/stockpricequote/'):
mcURLsplit = url.split('/')
mcSymbol = mcURLsplit[-1]
mcName = mcURLsplit[-2]
mcSector = mcURLsplit[-3]
annualFinancialURL = moneycontrolURL+'financials/'+mcName+'/results/yearly/'+mcSymbol+'#'+mcSymbol
def loadState(self):
self.filteredFinancialData = load('storedState.p')