def getCSVHistorialData(code,save = True,beginDate = '',endDate = str(date.today())):
from lxml import etree
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
begin = beginDate.split('-')
end = endDate.split('-')
period = '&d='+(str)(int(end[1])-1)+'&e='+end[2]+'&f='+end[0]+'&a='+(str)(int(begin[1])-1)+'&b='+begin[2]+'&c='+begin[0]
url = 'http://ichart.finance.yahoo.com/table.csv?s='+code2+period
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
page = parse(url).getroot()
logger.debug(page)
result = etree.tostring(page)
#print result
lines = result.split('\n')
from stocktrace.stock import Stock
historyDatas = []
for a in lines:
if a.find('html')!= -1:
continue
#print etree.tostring(tree)
datas = a.split(',')
#print datas
stock = Stock(code)
stock.date = datas[0]
stock.high = float(datas[2])
stock.low = float(datas[3])
stock.openPrice = float(datas[1])
stock.close = float(datas[4])
stock.volume = float(datas[5])
isNewData = True;
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock);
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item:item.date,reverse=True)
if (len(historyDatas) == 0):
logger.warning("No data downloaded for "+code)
else:
# update_week52(code)
logger.info(str(len(historyDatas))+" history Data downloaded for "+code)
def getHistorialData(code, save=True, beginDate='', endDate=str(date.today())):
from lxml import etree
from lxml.html import parse
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
# if len(endDate) == 0:
# print "Download all history date for "+code
# from datetime import date
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+str(date.today())+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
# else:
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22' + code2 + '%22%20and%20startDate%20%3D%20%22' + beginDate + '%22%20and%20endDate%20%3D%20%22' + endDate + '%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
# print lastStock
# print oldestStock
# print date.today().weekday()
# print beginDate
#check whether beginDate is weekday
begin = datetime.strptime(beginDate, '%Y-%m-%d').date()
beginWeekday = begin.weekday()
if (beginWeekday == 5):
delta = timedelta(2)
begin = begin + delta
elif (beginWeekday == 6):
delta = timedelta(1)
begin = begin + delta
# print begin.weekday()
beginDate = str(begin)
isUpdatedToDate = False
if lastStock is None and oldestStock is None:
print 'Begin to download history data for ' + code
else:
print code + " Current history data range " + oldestStock[
'date'] + "~" + lastStock['date']
isUpdatedToDate = (endDate <= lastStock['date']) and (
beginDate >= oldestStock['date'])
if isUpdatedToDate:
print "History data is updated to latest:" + code
return
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//quote')
historyDatas = []
for a in r:
tree = etree.ElementTree(a)
#print etree.tostring(tree)
stock = Stock(code)
stock.date = tree.xpath('//date')[0].text
stock.high = float(tree.xpath('//high')[0].text)
stock.low = float(tree.xpath('//low')[0].text)
stock.openPrice = float(tree.xpath('//open')[0].text)
stock.close = float(tree.xpath('//close')[0].text)
stock.volume = float(tree.xpath('//volume')[0].text)
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock)
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if (len(historyDatas) == 0):
print "No data downloaded for " + code
else:
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
# redclient.set(code,historyDatas[0].date)
# for stock in historyDatas:
# print stock
# pass
for i in range(len(historyDatas)):
if i == len(historyDatas) - 1:
continue
else:
last = historyDatas[i]
prev = historyDatas[i + 1]
if (last.openPrice != prev.close
and (last.low > prev.high or last.high < prev.low)):
#print "gap***"+last.__str__()
pass
def getHistorialData(code,save = True,beginDate = '',endDate = str(date.today())):
from lxml import etree
from lxml.html import parse
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
# if len(endDate) == 0:
# print "Download all history date for "+code
# from datetime import date
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+str(date.today())+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
# else:
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
# print lastStock
# print oldestStock
# print date.today().weekday()
# print beginDate
#check whether beginDate is weekday
begin = datetime.strptime(beginDate,'%Y-%m-%d').date()
beginWeekday = begin.weekday()
if (beginWeekday == 5):
delta = timedelta(2)
begin = begin+delta
elif (beginWeekday == 6):
delta = timedelta(1)
begin = begin+delta
# print begin.weekday()
beginDate = str(begin)
isUpdatedToDate = False
if lastStock is None and oldestStock is None:
print 'Begin to download history data for '+code
else:
print code+" Current history data range "+oldestStock['date']+"~"+lastStock['date']
isUpdatedToDate = (endDate <= lastStock['date']) and (beginDate >= oldestStock['date'])
if isUpdatedToDate:
print "History data is updated to latest:"+code
return
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//quote');
historyDatas = []
for a in r:
tree= etree.ElementTree(a)
#print etree.tostring(tree)
stock = Stock(code)
stock.date = tree.xpath('//date')[0].text
stock.high = float(tree.xpath('//high')[0].text)
stock.low = float(tree.xpath('//low')[0].text)
stock.openPrice = float(tree.xpath('//open')[0].text)
stock.close = float(tree.xpath('//close')[0].text)
stock.volume = float(tree.xpath('//volume')[0].text)
isNewData = True;
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock);
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item:item.date,reverse=True)
if (len(historyDatas) == 0):
print "No data downloaded for "+code
else:
logger.info(str(len(historyDatas))+" history Data downloaded for "+code)
# redclient.set(code,historyDatas[0].date)
# for stock in historyDatas:
# print stock
# pass
for i in range(len(historyDatas)):
if i == len(historyDatas)-1:
continue
else:
last = historyDatas[i]
prev = historyDatas[i+1]
if (last.openPrice!= prev.close and
(last.low >prev.high or last.high<prev.low)):
#print "gap***"+last.__str__()
pass
def getCSVHistorialData(code,
save=True,
beginDate='',
endDate=str(date.today())):
from lxml import etree
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
begin = beginDate.split('-')
end = endDate.split('-')
period = '&d=' + (
str)(int(end[1]) - 1) + '&e=' + end[2] + '&f=' + end[0] + '&a=' + (
str)(int(begin[1]) - 1) + '&b=' + begin[2] + '&c=' + begin[0]
url = 'http://ichart.finance.yahoo.com/table.csv?s=' + code2 + period
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
page = parse(url).getroot()
logger.debug(page)
result = etree.tostring(page)
#print result
lines = result.split('\n')
from stocktrace.stock import Stock
historyDatas = []
for a in lines:
if a.find('html') != -1:
continue
#print etree.tostring(tree)
datas = a.split(',')
#print datas
stock = Stock(code)
stock.date = datas[0]
stock.high = float(datas[2])
stock.low = float(datas[3])
stock.openPrice = float(datas[1])
stock.close = float(datas[4])
stock.volume = float(datas[5])
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock)
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if (len(historyDatas) == 0):
logger.warning("No data downloaded for " + code)
else:
# update_week52(code)
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
