class TestSequenceFunctions(unittest.TestCase):
code = '600583'
sh = Stock('600327')
sz = Stock('002236')
# 601318
def test_nhnl(self):
logger.debug(findByNhnl())
class TestSequenceFunctions(unittest.TestCase):
code = '600583'
sh = Stock('600327')
sz = Stock('002236')
# 601318
def test_sina_api(self):
logger.debug(getStock(self.code))
class XueQiuTestCase(unittest.TestCase):
code = 'SH600583'
sh = Stock('600327')
sz = Stock('SZ002236')
def test_print_stock(self):
xueqiuparser.parse_real_time('SH600583,SZ000728,GS,SH000001')
# xueqiuparser.parse_real_time('SH600011')
print(self.sh)
def test_download(self):
xueqiuparser.download_statistics()
def findTopN(top=20, condition=settings.HIGHER):
if condition == settings.HIGHER:
result = findByYearLow(top)
else:
result = findByYearHigh(top)
stocks = []
for stock in result:
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
s.name = stock.get('name')
if code.startswith('6'):
s.isInSh = True
#triggered = checkStockWithMA(code,40,10,condition)
#print triggered
stocks.append(s)
return stocks
def parse(code, parseIfeng=True, parseCap=True, parseSina=True):
from ifengparser import parseFinanceData
deli = '*************************************'
print deli
if (parseIfeng):
s = parseFinanceData(code)
else:
s = Stock(code)
if parseSina is False:
pass
else:
from sinaparser import getStock
s1 = getStock('sh' + code)
s.current = s1.current
s.percent = s1.percent
if parseCap:
from sseparser import parseCap
s2 = parseCap(code)
s.totalCap = s2.totalCap
s.floatingCap = s2.floatingCap
s.compute()
print s
return s
def get_real_time(code='600276'):
if code.startswith('60') or code.startswith('51'):
code = 'sh' + code
elif len(code) == 5:
code = 'hk' + code
else:
code = 'sz' + code
url = "http://hq.sinajs.cn/list=" + code
print('url:{}'.format(url))
r = requests.get(url)
print(r.content)
string = str(r.content, encoding="gbk")
test = string.split(',')
# print test
if code.startswith('hk'):
current = float(test[6])
else:
current = float(test[3])
yesterday = float(test[2])
high = float(test[4])
low = float(test[5])
volume = float(test[8])
if yesterday != 0:
percent = (current - yesterday) / yesterday * 100
else:
percent = 0
name = test[0].split('"')[1]
enc = "gbk"
# u_content = name.decode(enc) # decodes from enc to unicode
# utf8_name = u_content.encode("utf8")
stock = Stock(code, 0, current, percent, low, high, volume)
print(stock)
return stock
def download2(clearAll=False,
download_latest=False,
downloadHistory=False,
parse_industry=False,
stockList='stock_list_all'):
logger.info('Begin Download stock list data {}'.format(stockList))
if clearAll:
#clear redis cache
# redclient.flushall()
clear()
#download securities list from local
downloadQuoteList(True, False, stockList)
quotes = findAllExistentTickers()
import multiprocessing as mp
pool = mp.Pool(len(quotes))
for code in quotes:
s = Stock(code)
pool.apply_async(download_stock,
args=[
s, download_latest, settings.SINA, True,
settings.CSV_ENGINE, False
])
pool.close()
pool.join()
logger.info('****Download latest price from sina finished****')
def parseTickers(begin=600000, end=603366):
nonExistentTickers = findAllNonExistentTickers()
existingTickers = findAllExistentTickers()
print nonExistentTickers
print existingTickers
#parse shanghai tickers
for code in range(begin, end + 1):
if (str(code) in nonExistentTickers):
print 'Non-existent ticker***' + str(code)
continue
elif (str(code) in existingTickers):
print 'Existing ticker***' + str(code)
continue
code2 = str(code) + '.SS'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
print url
page = parse(url).getroot()
if page is None:
print "Fail to query for " + str(code)
return
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
stock = Stock(str(code))
if (errorMsg is None):
stock.name = ''
high = page.xpath('//dayshigh')[0].text
if (high is None):
print 'Non-existent ticker***' + str(code)
saveNonExistentTicker(stock)
continue
stock.high = float(high)
stock.low = float(page.xpath('//dayslow')[0].text)
stock.yearHigh = float(page.xpath('//yearhigh')[0].text)
stock.yearLow = float(page.xpath('//yearlow')[0].text)
close = page.xpath('//bid')[0].text
# print close
# print type(close)
if close is not None:
stock.close = float(close)
if (stock.high >= stock.yearHigh):
print 'stock trigger new high index*****' + code2
with io.open('nh.xml', 'wb') as f:
f.writelines(code2)
elif (stock.low <= stock.yearLow):
print 'stock trigger new low index*****' + code2
with io.open('nl.xml', 'wb') as f:
f.writelines(code2)
#print stock
saveTicker(stock)
#parse key statistics data from reuters
from reutersparser import parseKeyStatData
parseKeyStatData(code)
else:
saveNonExistentTicker(stock)
print 'Non-existent ticker***' + str(code)
print 'Finish parseTickers******'
def xueqiu(code='SH600036', access_token=xq_a_token):
if code.startswith('60') or code.startswith('51'):
code = 'SH' + code
elif len(code) == 5:
code = code
elif code == '999999' or code == '999998':
return Stock(code=code, current=1)
elif len(code) == 6:
code = 'SZ' + code
url = api_home + '/v4/stock/quote.json?code={}&_=1443253485389'
url = url.format(code)
print(url)
payload = {'access_token': access_token}
r = requests.get(url, params=payload, headers=headers)
# print r
# print r.json()
data = r.json().get(code)
print('data:{} code:{}'.format(data, code))
if data:
# Wed Dec 27 14:59:59 +0800 2017
stock = Stock(
code=code,
# name=data.get('name').encode("GB2312"),
current=data.get('current'),
percentage=data.get('percentage'),
open_price=data.get('open'),
high=data.get('high'),
low=data.get('low'),
close=data.get('close'),
low52week=data.get('low52week'),
high52week=data.get('high52week'),
change=data.get('change'),
pb=data.get('pb'))
time = data.get('time')
print(time)
if time:
time = arrow.get(time, 'ddd MMM DD HH:mm:ss Z YYYY')
print(time)
stock.date = time.datetime
print(stock)
return stock
else:
return None
def create_stock(request):
code = request.GET.get('code')
amount = request.GET.get('amount')
stock_tag = request.GET.get('tag')
print 'code:{},amount:{},tag:{}'.format(code, amount, stock_tag)
stock = Stock(code, amount)
stock.save()
# insert_stock(stock)
add_tag(code, 'top100')
return render_to_response('portfolio/index.html')
class TestSequenceFunctions(unittest.TestCase):
code = '600583'
sh = Stock('600327')
sz = Stock('002236')
# 601318
def test_peak_price(self):
logger.debug(findPeakStockByDays('000776', 50))
def test_peak_price(self):
delta = timedelta(-20)
begin = date.today() + delta
#begin_date ='2014-1-1'
#rise('600327',begin)
# find_percentage(['600327','600583'],'2014-01-01')
result = find_percentage(findAllExistentTickers(), '2014-01-01')
for stock in result:
logger.debug(stock.shortStr())
def stock_list():
import tushare as ts
df = ts.get_stock_basics()
stocks = df.index.tolist()
print((len(stocks)))
print(stocks)
for stock in stocks:
s = Stock()
s.code = stock
s.save()
def xueqiu(code='SH600036', access_token=xq_a_token):
if code.startswith('60') or code.startswith('51'):
code = 'SH' + code
elif len(code) == 5:
code = 'HK' + code
elif code == '999999' or code == '999998':
return Stock(code=code, current=1)
elif len(code) == 6:
code = 'SZ' + code
url = 'http://xueqiu.com/v4/stock/quote.json?code={}&_=1443253485389'
url = url.format(code)
payload = {'access_token': access_token}
r = requests.get(url, params=payload, headers=headers)
# print r
# print r.json()
data = r.json().get(code)
# print data
time = data.get('time')
import arrow
time = arrow.get(time, 'ddd MMM DD HH:mm:ss Z YYYY')
# print time
stock = Stock(code=code,
name=data.get('name').encode("GB2312"),
current=data.get('current'),
percentage=data.get('percentage'),
open_price=data.get('open'),
high=data.get('high'),
low=data.get('low'),
close=data.get('close'),
low52week=data.get('low52week'),
high52week=data.get('high52week'),
pe_lyr=data.get('pe_lyr'),
pb=data.get('pb'),
date=time)
# print stock
return stock
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
url = 'http://app.finance.ifeng.com/data/stock/cwjk.php?symbol=' + code
print url
page = parse(url).getroot()
result = etree.tostring(page)
#print result
import io
with io.open('test.xml', 'wb') as f:
#f.writelines(result)
pass
r = page.xpath('//div[@class="tab01"]')
#print len(r)
from stocktrace.stock import Stock
stock = Stock(code)
for a in r:
tree = etree.ElementTree(a)
#print etree.tostring(tree)
datas = tree.xpath('//td')
#print len(datas)
index = 0
for data in datas:
dataTree = etree.ElementTree(data)
#print etree.tostring(dataTree)
values = dataTree.xpath('//text()')
index += 1
#print index
if (len(values) == 1):
#print values
#print len(values[0])
#print str(values[0])
if (index == 32):
mgsy = values[0]
#print mgsy+'***************'
stock.mgsy = mgsy
elif (index == 52):
mgjzc = values[0]
#print mgjzc+'***************'
stock.mgjzc = mgjzc
elif (index == 2):
last_update = values[0]
#print last_update
stock.lastUpdate = last_update
return stock
def find_percentage(code_list, from_date):
# history = find_history_by_date(code,from_date)
history = find_all_history_by_date(from_date)
# size = history.count()
# logger.debug(size)
#logger.debug(history[0])
#logger.debug(history[size-1])
#for item in history:
# logger.debug(item)
df = pd.DataFrame(list(history))
# logger.debug(df)
result = []
for code in code_list:
df_by_code = df.loc[df['code'] == code]
# logger.debug(df_by_code)
# logger.debug('df shape:{}'.format(df_by_code.shape))
logger.debug('code:{} df length:{}'.format(code,
len(df_by_code.index)))
# size = len(df_by_code)
# logger.debug('df length:{}'.format(size))
# high_week52_index = 0
# low_week52_index = size-1
# end_close = df_by_code[['date','close']][high_week52_index:high_week52_index+1]
# begin_close = df_by_code[['date','close']][low_week52_index:low_week52_index+1]
# logger.debug(begin_close)
# logger.debug(end_close)
# logger.debug(begin_close['close'])
# logger.debug(end_close['close'])
# logger.debug(type(begin_close['close']))
if len(df_by_code.index) == 0:
continue
end_close = df_by_code.iloc[0]['close']
begin_close = df_by_code.iloc[-1]['close']
# logger.debug(begin_close)
# logger.debug(end_close)
code_percentage = (float(end_close) -
float(begin_close)) / float(begin_close)
logger.debug('code:{}:{}'.format(code, code_percentage))
stock = Stock(code)
stock.percent = code_percentage
result.append(stock)
result = sorted(result, key=lambda s: s.percent)
logger.debug(result)
return result
def findQuoteByCode(code, condition=settings.HIGHER):
historyDatas = db.tickers
stock = historyDatas.find_one({"code": code})
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
if code.startswith('6'):
s.isInSh = True
return s
class TestSequenceFunctions(unittest.TestCase):
code = '600583'
sh = Stock('600327')
sz = Stock('002236')
# 601318
def test_print_stock(self):
print self.sh
def test_remove_stock(self):
remove_stock(self.sh.code)
def test_download_sh(self):
download_stock(self.sh,
download_latest=True,
realtime_engine=settings.SINA,
download_history=True,
history_engine=settings.CSV_ENGINE,
download_statistics=False)
def test_download_sz(self):
download_stock(self.sz,
download_latest=True,
realtime_engine=settings.YAHOO,
download_history=True,
history_engine=settings.YAHOO,
download_statistics=False)
def test_sina_latest(self):
update(self.code, engine='sina')
def test_ydn_latest(self):
update(self.code, engine='yahoo')
def test_download_hold(self):
download2(clearAll=True,
download_latest=True,
downloadHistory=True,
parse_industry=False,
stockList=settings.STOCK_LIST_TOP100)
def test_download_top100(self):
download2(clearAll=True,
download_latest=True,
downloadHistory=True,
parse_industry=False,
stockList=settings.STOCK_LIST_TOP100)
def test_clear(self):
from stocktrace.dao.stockdao import clear
clear()
def test_poll_ydn(self):
download_history_data(self.code, True, engine='ydn')
def test_yahoo_csv(self):
download_history_data(self.code, True)
def test_yahoo_csv2(self):
url = 'http://ichart.finance.yahoo.com/table.csv?s=601318.SS&d=2&e=08&f=2014&a=0&b=01&c=2012'
import requests
r = requests.get(url)
r.status_code
logger.debug(r.text)
# downloadHistorialData(self.code,True)
def test_week52(self):
history = find_week52_history(self.code)
logger.debug(history)
logger.debug(history.count())
# for h in history:
# logger.debug(h)
def test_update_week52(self):
update_week52(self.code)
def test_yahoo(self):
update(self.code, engine='sina')
def test_pandas(self):
import pandas as pd
# result = findStockByCode('600327')
result = find_week52_history(self.code)
logger.debug(result)
df = pd.DataFrame(list(result))
logger.debug(df)
logger.debug(df.shape)
logger.debug(df['low'].min())
logger.debug(df['low'].argmin())
logger.debug(df['high'].max())
high_week52_index = df['high'].argmax()
low_week52_index = df['low'].argmin()
logger.debug(high_week52_index)
logger.debug(df[['date',
'high']][high_week52_index:high_week52_index + 1])
logger.debug(df[['date',
'low']][low_week52_index:low_week52_index + 1])
def test_tasks(self):
from stocktrace.task.tasks import add
result = add.delay(4, 4)
print result
print(result.get())
# logger.debug(str(begin_close))
# logger.debug('open:{}'.format(df_by_code[0,'close']))
# logger.debug('close:{}'.format(df_by_code[low_week52_index:low_week52_index+1]['close']))
def df_to_collection(df, collection):
records = json.loads(df.T.to_json()).values()
# print records
#TODO get_collection is not supported in pymongo 2.8
# db.get_collection(collection).insert(records)
db.sw.insert(records)
if __name__ == '__main__':
from stocktrace.stock import Stock
stock = Stock('600880')
# print findQuoteByCode('600327').yearHighLow()
# print findQuoteByCode('600327',settings.LOWER).yearHighLow()
#findAllNonExistentTickers()
#updateTickerWithKeyStats('600004',14.00,2.36,0.56,2333.5)
# stocks = findAllExistentTickers()
# for stock in stocks:
# updateTickerWithKeyStats(stock,14.00,2.36,0.56,2333.5)
#print getMarketPe();
# print getAvgPe();
# print getAvgPb();
# print getPbLessThan1()
# stocks =['600000', '600004', '600005']
# batchInsertTicker(stocks)
# print findLastUpdate('600890')
# print countByCode('600655')
if (stockList is None):
return stocks
result = []
stocksInRedis = redclient.zrange(stockList,0,-1)
if (len(stocksInRedis) == 0):
return stocks
for stock in stocks:
if any(stock in s for s in stocksInRedis):
logger.debug(str(stock)+'already in '+stockList)
continue
else:
result.append(stock)
return result
def findStocksByList(stockList):
stocksInRedis = redclient.zrange(stockList,0,-1)
return stocksInRedis
if __name__ == '__main__':
stocks = []
from stocktrace.stock import Stock
stocks.append(Stock('600327'))
stocks.append(Stock('601111'))
stocks.append(Stock('600221'))
result = filterStocksByIndustry(stocks,'航空运输业')
for stock in result:
print stock
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&diagnostics=true'
logger.debug(url)
try:
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
if (errorMsg is None):
print 'OK'
stock = Stock(code)
stock.name = ''
else:
print 'error'
quote = page.xpath('/html/body/query/results/quote')
#print len(quote)
#print len(page.xpath('//ask'))
#print page.xpath('/html/body/query/results/quote/ask[1]/text()')
#print page.xpath('//ask[1]/text()')[0]#both works
yearLow = page.xpath('//yearlow[1]/text()')[0]
logger.debug('yearLow' + yearLow)
yearHigh = page.xpath('//yearhigh[1]/text()')[0]
print 'yearHigh' + yearHigh
PercentChangeFromYearLow = page.xpath(
'//percentchangefromyearlow[1]/text()')[0]
print 'PercentChangeFromYearLow' + PercentChangeFromYearLow
PercebtChangeFromYearHigh = page.xpath(
'//percebtchangefromyearhigh[1]/text()')[0]
print 'PercebtChangeFromYearHigh' + PercebtChangeFromYearHigh
FiftydayMovingAverage = page.xpath(
'//fiftydaymovingaverage[1]/text()')[0]
print 'FiftydayMovingAverage' + FiftydayMovingAverage
TwoHundreddayMovingAverage = page.xpath(
'//twohundreddaymovingaverage[1]/text()')[0]
print 'TwoHundreddayMovingAverage' + TwoHundreddayMovingAverage
PercentChangeFromTwoHundreddayMovingAverage = page.xpath(
'//percentchangefromtwohundreddaymovingaverage[1]/text()')[0]
print 'PercentChangeFromTwoHundreddayMovingAverage' + PercentChangeFromTwoHundreddayMovingAverage
PercentChangeFromFiftydayMovingAverage = page.xpath(
'//percentchangefromfiftydaymovingaverage[1]/text()')[0]
print 'PercentChangeFromFiftydayMovingAverage' + PercentChangeFromFiftydayMovingAverage
logger.debug(page.xpath('//DaysLow[1]/text()'))
LastTradePriceOnly = page.xpath('//lasttradepriceonly[1]/text()')[0]
stock = Stock(code)
stock.yearHigh = float(yearHigh)
stock.yearLow = float(yearLow)
#stock.PercebtChangeFromYearHigh = float(PercebtChangeFromYearHigh.rstrip('%'))
stock.PercentChangeFromYearHigh = float(
PercebtChangeFromYearHigh.lstrip('-').rstrip('%'))
stock.PercentChangeFromYearLow = float(
PercentChangeFromYearLow.lstrip('+').rstrip('%'))
stock.ma50 = float(FiftydayMovingAverage)
stock.ma200 = float(TwoHundreddayMovingAverage)
if LastTradePriceOnly is not None:
logger.debug("{} current: {}".format(code, LastTradePriceOnly))
stock.current = float(LastTradePriceOnly)
close = page.xpath('//bid')[0].text
if close is not None:
stock.close = float(close)
return stock
except Exception as ex:
logger.error('Fail to download latest update from yahoo API:' + code)
traceback.print_exc(file=sys.stdout)
logger.exception(ex)
return None
def getCSVHistorialData(code='600276',
save=True,
beginDate='',
endDate=str(date.today())):
from lxml import etree
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if len(code) == 9:
code2 = code
elif code.startswith('6'):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
begin = beginDate.split('-')
end = endDate.split('-')
print 'begin:{} end:{}'.format(begin, end)
period = '&d=' + (
str)(int(end[1]) - 1) + '&e=' + end[2] + '&f=' + end[0] + '&a=' + (
str)(int(begin[1]) - 1) + '&b=' + begin[2] + '&c=' + begin[0]
url = 'http://ichart.finance.yahoo.com/table.csv?s=' + code2 + period
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
page = parse(url).getroot()
if page is None:
logger.error('Fail to download history data for:{}'.format(url))
logger.debug(page)
result = etree.tostring(page)
#print result
lines = result.split('\n')
from stocktrace.stock import Stock
historyDatas = []
for a in lines:
if a.find('html') != -1:
continue
#print etree.tostring(tree)
datas = a.split(',')
#print datas
stock = Stock(code)
stock.date = datas[0]
stock.high = float(datas[2])
stock.low = float(datas[3])
stock.open_price = float(datas[1])
stock.close = float(datas[4])
stock.volume = float(datas[5])
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
# saveStock(stock)
stock.save()
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if len(historyDatas) == 0:
logger.warning("No data downloaded for " + code)
else:
# update_week52(code)
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
def getStock(code):
#Sina API
if (code.startswith('60')):
code = 'sh'+code
else:
code = 'sz'+code
url = "http://hq.sinajs.cn/list="+code
logger.debug(url)
#Google Finance API
#url = 'http://finance.google.com/finance/info?q=600030'
import urllib2,os,sys
pg = urllib2.urlopen(url)
cont = pg.read()
#print cont
test = cont.split(',')
if (len(test) <9):
logger.error('error passe:'+code)
return
yesterday = float(test[2])
current = float(test[3])
high = float(test[4])
low = float(test[5])
volume = float(test[8])
percent = (current-yesterday)/yesterday*100
name = test[0].split('"')[1]
logger.debug(name)
# logger.debug(str(name))
enc = "gbk"
u_content = name.decode(enc) # decodes from enc to unicode
utf8_name = u_content.encode("utf8")
logger.debug(utf8_name)
stock = Stock(code,current,percent,low,high,volume)
stock.name = utf8_name
#check threshold
if (current == 0.0):
pass
elif (percent >=2):
#Alarm
stock.alert = True;
stock.state = 'UP'
elif (percent <=-5):
#Alarm
stock.alert = True;
stock.state = 'CRITICAL'
elif (percent <=-2):
#Alarm
stock.alert = True;
stock.state = 'WARNING'
elif (stock.code == 'sh000001' and (percent >=1 or percent <=-1)):
stock.state = 'WARNING'
#check whether state changed
state = stateCache.get(code)
#print state
if (state!= stock.state):
stateCache[code] = stock.state;
pass
else:
stock.alert = False;
logger.info( '****Download latest price from SINA finished****'+stock.code)
#logger.info(stock)
return stock
def getHistorialData(code, save=True, beginDate='', endDate=str(date.today())):
from lxml import etree
from lxml.html import parse
#yahoo stock ticker need post-fix ".SS" for Shanghai,'.SZ' for shenzheng
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
# if len(endDate) == 0:
# print "Download all history date for "+code
# from datetime import date
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+str(date.today())+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
# else:
# url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22'+code2+'%22%20and%20startDate%20%3D%20%22'+beginDate+'%22%20and%20endDate%20%3D%20%22'+endDate+'%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.historicaldata%20where%20symbol%20%3D%20%22' + code2 + '%22%20and%20startDate%20%3D%20%22' + beginDate + '%22%20and%20endDate%20%3D%20%22' + endDate + '%22&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
logger.debug(url)
#check whether data is update to latest
#from stocktrace.dao.stockdao import findLastUpdate
#from stocktrace.dao.stockdao import findOldestUpdate
lastStock = findLastUpdate(code)
oldestStock = findOldestUpdate(code)
# print lastStock
# print oldestStock
# print date.today().weekday()
# print beginDate
#check whether beginDate is weekday
begin = datetime.strptime(beginDate, '%Y-%m-%d').date()
beginWeekday = begin.weekday()
if (beginWeekday == 5):
delta = timedelta(2)
begin = begin + delta
elif (beginWeekday == 6):
delta = timedelta(1)
begin = begin + delta
# print begin.weekday()
beginDate = str(begin)
isUpdatedToDate = False
if lastStock is None and oldestStock is None:
print 'Begin to download history data for ' + code
else:
print code + " Current history data range " + oldestStock[
'date'] + "~" + lastStock['date']
isUpdatedToDate = (endDate <= lastStock['date']) and (
beginDate >= oldestStock['date'])
if isUpdatedToDate:
print "History data is updated to latest:" + code
return
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//quote')
historyDatas = []
for a in r:
tree = etree.ElementTree(a)
#print etree.tostring(tree)
stock = Stock(code)
stock.date = tree.xpath('//date')[0].text
stock.high = float(tree.xpath('//high')[0].text)
stock.low = float(tree.xpath('//low')[0].text)
stock.openPrice = float(tree.xpath('//open')[0].text)
stock.close = float(tree.xpath('//close')[0].text)
stock.volume = float(tree.xpath('//volume')[0].text)
isNewData = True
if lastStock is not None:
isNewData = (stock.date > lastStock['date']) or (
stock.date < oldestStock['date'])
#print stock.date+'***isNewData***'+str(isNewData)
if isNewData and save:
saveStock(stock)
#print stock
historyDatas.append(stock)
historyDatas.sort(key=lambda item: item.date, reverse=True)
if (len(historyDatas) == 0):
print "No data downloaded for " + code
else:
logger.info(
str(len(historyDatas)) + " history Data downloaded for " + code)
# redclient.set(code,historyDatas[0].date)
# for stock in historyDatas:
# print stock
# pass
for i in range(len(historyDatas)):
if i == len(historyDatas) - 1:
continue
else:
last = historyDatas[i]
prev = historyDatas[i + 1]
if (last.openPrice != prev.close
and (last.low > prev.high or last.high < prev.low)):
#print "gap***"+last.__str__()
pass
