def parseTickers(begin=600000,end=603366):
nonExistentTickers = findAllNonExistentTickers()
existingTickers = findAllExistentTickers()
print nonExistentTickers
print existingTickers
#parse shanghai tickers
for code in range(begin,end+1):
if (str(code) in nonExistentTickers):
print 'Non-existent ticker***'+str(code)
continue;
elif (str(code) in existingTickers):
print 'Existing ticker***'+str(code)
continue;
code2 = str(code) +'.SS'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22'+code2+'%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
print url
page = parse(url).getroot()
if page is None:
print "Fail to query for "+str(code)
return
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid');
errorMsg = r[0].text
stock = Stock(str(code))
if (errorMsg is None):
stock.name = ''
high = page.xpath('//dayshigh')[0].text
if (high is None):
print 'Non-existent ticker***'+str(code)
saveNonExistentTicker(stock)
continue;
stock.high = float(high)
stock.low = float(page.xpath('//dayslow')[0].text)
stock.yearHigh = float(page.xpath('//yearhigh')[0].text)
stock.yearLow = float(page.xpath('//yearlow')[0].text)
close = page.xpath('//bid')[0].text;
# print close
# print type(close)
if close is not None:
stock.close = float(close)
if (stock.high>=stock.yearHigh):
print 'stock trigger new high index*****'+code2
with io.open('nh.xml','wb') as f:
f.writelines(code2)
elif (stock.low <=stock.yearLow):
print 'stock trigger new low index*****'+code2
with io.open('nl.xml','wb') as f:
f.writelines(code2)
#print stock
saveTicker(stock)
#parse key statistics data from reuters
from reutersparser import parseKeyStatData
parseKeyStatData(code)
else:
saveNonExistentTicker(stock)
print 'Non-existent ticker***'+str(code)
print 'Finish parseTickers******'
def parseTickers(begin=600000, end=603366):
nonExistentTickers = findAllNonExistentTickers()
existingTickers = findAllExistentTickers()
print nonExistentTickers
print existingTickers
#parse shanghai tickers
for code in range(begin, end + 1):
if (str(code) in nonExistentTickers):
print 'Non-existent ticker***' + str(code)
continue
elif (str(code) in existingTickers):
print 'Existing ticker***' + str(code)
continue
code2 = str(code) + '.SS'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
print url
page = parse(url).getroot()
if page is None:
print "Fail to query for " + str(code)
return
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
stock = Stock(str(code))
if (errorMsg is None):
stock.name = ''
high = page.xpath('//dayshigh')[0].text
if (high is None):
print 'Non-existent ticker***' + str(code)
saveNonExistentTicker(stock)
continue
stock.high = float(high)
stock.low = float(page.xpath('//dayslow')[0].text)
stock.yearHigh = float(page.xpath('//yearhigh')[0].text)
stock.yearLow = float(page.xpath('//yearlow')[0].text)
close = page.xpath('//bid')[0].text
# print close
# print type(close)
if close is not None:
stock.close = float(close)
if (stock.high >= stock.yearHigh):
print 'stock trigger new high index*****' + code2
with io.open('nh.xml', 'wb') as f:
f.writelines(code2)
elif (stock.low <= stock.yearLow):
print 'stock trigger new low index*****' + code2
with io.open('nl.xml', 'wb') as f:
f.writelines(code2)
#print stock
saveTicker(stock)
#parse key statistics data from reuters
from reutersparser import parseKeyStatData
parseKeyStatData(code)
else:
saveNonExistentTicker(stock)
print 'Non-existent ticker***' + str(code)
print 'Finish parseTickers******'
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22'+code2+'%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&diagnostics=true'
logger.debug(url)
try:
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid');
errorMsg = r[0].text
if (errorMsg is None):
print 'OK'
stock = Stock(code)
stock.name = ''
else:
print 'error'
quote = page.xpath('/html/body/query/results/quote');
#print len(quote)
#print len(page.xpath('//ask'))
#print page.xpath('/html/body/query/results/quote/ask[1]/text()')
#print page.xpath('//ask[1]/text()')[0]#both works
yearLow = page.xpath('//yearlow[1]/text()')[0]
logger.debug('yearLow'+yearLow)
yearHigh = page.xpath('//yearhigh[1]/text()')[0]
print 'yearHigh'+yearHigh
PercentChangeFromYearLow = page.xpath('//percentchangefromyearlow[1]/text()')[0]
print 'PercentChangeFromYearLow'+PercentChangeFromYearLow
PercebtChangeFromYearHigh = page.xpath('//percebtchangefromyearhigh[1]/text()')[0]
print 'PercebtChangeFromYearHigh'+PercebtChangeFromYearHigh
FiftydayMovingAverage = page.xpath('//fiftydaymovingaverage[1]/text()')[0]
print 'FiftydayMovingAverage'+FiftydayMovingAverage
TwoHundreddayMovingAverage = page.xpath('//twohundreddaymovingaverage[1]/text()')[0]
print 'TwoHundreddayMovingAverage'+TwoHundreddayMovingAverage
PercentChangeFromTwoHundreddayMovingAverage = page.xpath('//percentchangefromtwohundreddaymovingaverage[1]/text()')[0]
print 'PercentChangeFromTwoHundreddayMovingAverage'+PercentChangeFromTwoHundreddayMovingAverage
PercentChangeFromFiftydayMovingAverage = page.xpath('//percentchangefromfiftydaymovingaverage[1]/text()')[0]
print 'PercentChangeFromFiftydayMovingAverage'+PercentChangeFromFiftydayMovingAverage
logger.debug(page.xpath('//DaysLow[1]/text()'))
LastTradePriceOnly = page.xpath('//lasttradepriceonly[1]/text()')[0]
stock = Stock(code)
stock.yearHigh = float(yearHigh)
stock.yearLow = float(yearLow)
#stock.PercebtChangeFromYearHigh = float(PercebtChangeFromYearHigh.rstrip('%'))
stock.PercentChangeFromYearHigh = float(PercebtChangeFromYearHigh.lstrip('-').rstrip('%'))
stock.PercentChangeFromYearLow = float(PercentChangeFromYearLow.lstrip('+').rstrip('%'))
stock.ma50 = float(FiftydayMovingAverage)
stock.ma200 = float(TwoHundreddayMovingAverage)
if LastTradePriceOnly is not None:
logger.debug("{} current: {}".format(code,LastTradePriceOnly))
stock.current = float(LastTradePriceOnly)
close = page.xpath('//bid')[0].text;
if close is not None:
stock.close = float(close)
return stock
except Exception as ex:
logger.error('Fail to download latest update from yahoo API:'+code)
traceback.print_exc(file=sys.stdout)
logger.exception(ex)
return None
def getCSVHistorialData(code,save = True,beginDate = '',endDate = str(date.today())):
from lxml import etree
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
begin = beginDate.split('-')
end = endDate.split('-')
period = '&d='+(str)(int(end[1])-1)+'&e='+end[2]+'&f='+end[0]+'&a='+(str)(int(begin[1])-1)+'&b='+begin[2]+'&c='+begin[0]
url = 'http://ichart.finance.yahoo.com/table.csv?s='+code2+period
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
page = parse(url).getroot()
logger.debug(page)
result = etree.tostring(page)
#print result
lines = result.split('\n')
from stocktrace.stock import Stock
historyDatas = []
for a in lines:
if a.find('html')!= -1:
continue
#print etree.tostring(tree)
datas = a.split(',')
#print datas
stock = Stock(code)
stock.date = datas[0]
stock.high = float(datas[2])
stock.low = float(datas[3])
stock.openPrice = float(datas[1])
stock.close = float(datas[4])
stock.volume = float(datas[5])
isNewData = True;
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock);
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item:item.date,reverse=True)
if (len(historyDatas) == 0):
logger.warning("No data downloaded for "+code)
else:
# update_week52(code)
logger.info(str(len(historyDatas))+" history Data downloaded for "+code)
def getHistorialData(code,save = True,beginDate = '',endDate = str(date.today())):
from lxml import etree
from lxml.html import parse
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
# if len(endDate) == 0:
# print "Download all history date for "+code
# from datetime import date
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+str(date.today())+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
# else:
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
# print lastStock
# print oldestStock
# print date.today().weekday()
# print beginDate
#check whether beginDate is weekday
begin = datetime.strptime(beginDate,'%Y-%m-%d').date()
beginWeekday = begin.weekday()
if (beginWeekday == 5):
delta = timedelta(2)
begin = begin+delta
elif (beginWeekday == 6):
delta = timedelta(1)
begin = begin+delta
# print begin.weekday()
beginDate = str(begin)
isUpdatedToDate = False
if lastStock is None and oldestStock is None:
print 'Begin to download history data for '+code
else:
print code+" Current history data range "+oldestStock['date']+"~"+lastStock['date']
isUpdatedToDate = (endDate <= lastStock['date']) and (beginDate >= oldestStock['date'])
if isUpdatedToDate:
print "History data is updated to latest:"+code
return
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//quote');
historyDatas = []
for a in r:
tree= etree.ElementTree(a)
#print etree.tostring(tree)
stock = Stock(code)
stock.date = tree.xpath('//date')[0].text
stock.high = float(tree.xpath('//high')[0].text)
stock.low = float(tree.xpath('//low')[0].text)
stock.openPrice = float(tree.xpath('//open')[0].text)
stock.close = float(tree.xpath('//close')[0].text)
stock.volume = float(tree.xpath('//volume')[0].text)
isNewData = True;
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock);
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item:item.date,reverse=True)
if (len(historyDatas) == 0):
print "No data downloaded for "+code
else:
logger.info(str(len(historyDatas))+" history Data downloaded for "+code)
# redclient.set(code,historyDatas[0].date)
# for stock in historyDatas:
# print stock
# pass
for i in range(len(historyDatas)):
if i == len(historyDatas)-1:
continue
else:
last = historyDatas[i]
prev = historyDatas[i+1]
if (last.openPrice!= prev.close and
(last.low >prev.high or last.high<prev.low)):
#print "gap***"+last.__str__()
pass
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&diagnostics=true'
logger.debug(url)
try:
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
if (errorMsg is None):
print 'OK'
stock = Stock(code)
stock.name = ''
else:
print 'error'
quote = page.xpath('/html/body/query/results/quote')
#print len(quote)
#print len(page.xpath('//ask'))
#print page.xpath('/html/body/query/results/quote/ask[1]/text()')
#print page.xpath('//ask[1]/text()')[0]#both works
yearLow = page.xpath('//yearlow[1]/text()')[0]
logger.debug('yearLow' + yearLow)
yearHigh = page.xpath('//yearhigh[1]/text()')[0]
print 'yearHigh' + yearHigh
PercentChangeFromYearLow = page.xpath(
'//percentchangefromyearlow[1]/text()')[0]
print 'PercentChangeFromYearLow' + PercentChangeFromYearLow
PercebtChangeFromYearHigh = page.xpath(
'//percebtchangefromyearhigh[1]/text()')[0]
print 'PercebtChangeFromYearHigh' + PercebtChangeFromYearHigh
FiftydayMovingAverage = page.xpath(
'//fiftydaymovingaverage[1]/text()')[0]
print 'FiftydayMovingAverage' + FiftydayMovingAverage
TwoHundreddayMovingAverage = page.xpath(
'//twohundreddaymovingaverage[1]/text()')[0]
print 'TwoHundreddayMovingAverage' + TwoHundreddayMovingAverage
PercentChangeFromTwoHundreddayMovingAverage = page.xpath(
'//percentchangefromtwohundreddaymovingaverage[1]/text()')[0]
print 'PercentChangeFromTwoHundreddayMovingAverage' + PercentChangeFromTwoHundreddayMovingAverage
PercentChangeFromFiftydayMovingAverage = page.xpath(
'//percentchangefromfiftydaymovingaverage[1]/text()')[0]
print 'PercentChangeFromFiftydayMovingAverage' + PercentChangeFromFiftydayMovingAverage
logger.debug(page.xpath('//DaysLow[1]/text()'))
LastTradePriceOnly = page.xpath('//lasttradepriceonly[1]/text()')[0]
stock = Stock(code)
stock.yearHigh = float(yearHigh)
stock.yearLow = float(yearLow)
#stock.PercebtChangeFromYearHigh = float(PercebtChangeFromYearHigh.rstrip('%'))
stock.PercentChangeFromYearHigh = float(
PercebtChangeFromYearHigh.lstrip('-').rstrip('%'))
stock.PercentChangeFromYearLow = float(
PercentChangeFromYearLow.lstrip('+').rstrip('%'))
stock.ma50 = float(FiftydayMovingAverage)
stock.ma200 = float(TwoHundreddayMovingAverage)
if LastTradePriceOnly is not None:
logger.debug("{} current: {}".format(code, LastTradePriceOnly))
stock.current = float(LastTradePriceOnly)
close = page.xpath('//bid')[0].text
if close is not None:
stock.close = float(close)
return stock
except Exception as ex:
logger.error('Fail to download latest update from yahoo API:' + code)
traceback.print_exc(file=sys.stdout)
logger.exception(ex)
return None
def getCSVHistorialData(code='600276',
save=True,
beginDate='',
endDate=str(date.today())):
from lxml import etree
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if len(code) == 9:
code2 = code
elif code.startswith('6'):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
begin = beginDate.split('-')
end = endDate.split('-')
print 'begin:{} end:{}'.format(begin, end)
period = '&d=' + (
str)(int(end[1]) - 1) + '&e=' + end[2] + '&f=' + end[0] + '&a=' + (
str)(int(begin[1]) - 1) + '&b=' + begin[2] + '&c=' + begin[0]
url = 'http://ichart.finance.yahoo.com/table.csv?s=' + code2 + period
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
page = parse(url).getroot()
if page is None:
logger.error('Fail to download history data for:{}'.format(url))
logger.debug(page)
result = etree.tostring(page)
#print result
lines = result.split('\n')
from stocktrace.stock import Stock
historyDatas = []
for a in lines:
if a.find('html') != -1:
continue
#print etree.tostring(tree)
datas = a.split(',')
#print datas
stock = Stock(code)
stock.date = datas[0]
stock.high = float(datas[2])
stock.low = float(datas[3])
stock.open_price = float(datas[1])
stock.close = float(datas[4])
stock.volume = float(datas[5])
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
# saveStock(stock)
stock.save()
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if len(historyDatas) == 0:
logger.warning("No data downloaded for " + code)
else:
# update_week52(code)
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
def getHistorialData(code, save=True, beginDate='', endDate=str(date.today())):
from lxml import etree
from lxml.html import parse
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
# if len(endDate) == 0:
# print "Download all history date for "+code
# from datetime import date
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+str(date.today())+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
# else:
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22' + code2 + '%22%20and%20startDate%20%3D%20%22' + beginDate + '%22%20and%20endDate%20%3D%20%22' + endDate + '%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
# print lastStock
# print oldestStock
# print date.today().weekday()
# print beginDate
#check whether beginDate is weekday
begin = datetime.strptime(beginDate, '%Y-%m-%d').date()
beginWeekday = begin.weekday()
if (beginWeekday == 5):
delta = timedelta(2)
begin = begin + delta
elif (beginWeekday == 6):
delta = timedelta(1)
begin = begin + delta
# print begin.weekday()
beginDate = str(begin)
isUpdatedToDate = False
if lastStock is None and oldestStock is None:
print 'Begin to download history data for ' + code
else:
print code + " Current history data range " + oldestStock[
'date'] + "~" + lastStock['date']
isUpdatedToDate = (endDate <= lastStock['date']) and (
beginDate >= oldestStock['date'])
if isUpdatedToDate:
print "History data is updated to latest:" + code
return
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//quote')
historyDatas = []
for a in r:
tree = etree.ElementTree(a)
#print etree.tostring(tree)
stock = Stock(code)
stock.date = tree.xpath('//date')[0].text
stock.high = float(tree.xpath('//high')[0].text)
stock.low = float(tree.xpath('//low')[0].text)
stock.openPrice = float(tree.xpath('//open')[0].text)
stock.close = float(tree.xpath('//close')[0].text)
stock.volume = float(tree.xpath('//volume')[0].text)
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock)
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if (len(historyDatas) == 0):
print "No data downloaded for " + code
else:
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
# redclient.set(code,historyDatas[0].date)
# for stock in historyDatas:
# print stock
# pass
for i in range(len(historyDatas)):
if i == len(historyDatas) - 1:
continue
else:
last = historyDatas[i]
prev = historyDatas[i + 1]
if (last.openPrice != prev.close
and (last.low > prev.high or last.high < prev.low)):
#print "gap***"+last.__str__()
pass
