def findTopN(top=20,condition=settings.HIGHER):
if condition == settings.HIGHER:
result = findByYearLow(top)
else:
result = findByYearHigh(top)
stocks = []
for stock in result:
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
s.name = stock.get('name')
if code.startswith('6'):
s.isInSh = True;
#triggered = checkStockWithMA(code,40,10,condition)
#print triggered
stocks.append(s)
return stocks
def findTopN(top=20, condition=settings.HIGHER):
if condition == settings.HIGHER:
result = findByYearLow(top)
else:
result = findByYearHigh(top)
stocks = []
for stock in result:
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
s.name = stock.get('name')
if code.startswith('6'):
s.isInSh = True
#triggered = checkStockWithMA(code,40,10,condition)
#print triggered
stocks.append(s)
return stocks
def parseTickers(begin=600000,end=603366):
nonExistentTickers = findAllNonExistentTickers()
existingTickers = findAllExistentTickers()
print nonExistentTickers
print existingTickers
#parse shanghai tickers
for code in range(begin,end+1):
if (str(code) in nonExistentTickers):
print 'Non-existent ticker***'+str(code)
continue;
elif (str(code) in existingTickers):
print 'Existing ticker***'+str(code)
continue;
code2 = str(code) +'.SS'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22'+code2+'%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
print url
page = parse(url).getroot()
if page is None:
print "Fail to query for "+str(code)
return
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid');
errorMsg = r[0].text
stock = Stock(str(code))
if (errorMsg is None):
stock.name = ''
high = page.xpath('//dayshigh')[0].text
if (high is None):
print 'Non-existent ticker***'+str(code)
saveNonExistentTicker(stock)
continue;
stock.high = float(high)
stock.low = float(page.xpath('//dayslow')[0].text)
stock.yearHigh = float(page.xpath('//yearhigh')[0].text)
stock.yearLow = float(page.xpath('//yearlow')[0].text)
close = page.xpath('//bid')[0].text;
# print close
# print type(close)
if close is not None:
stock.close = float(close)
if (stock.high>=stock.yearHigh):
print 'stock trigger new high index*****'+code2
with io.open('nh.xml','wb') as f:
f.writelines(code2)
elif (stock.low <=stock.yearLow):
print 'stock trigger new low index*****'+code2
with io.open('nl.xml','wb') as f:
f.writelines(code2)
#print stock
saveTicker(stock)
#parse key statistics data from reuters
from reutersparser import parseKeyStatData
parseKeyStatData(code)
else:
saveNonExistentTicker(stock)
print 'Non-existent ticker***'+str(code)
print 'Finish parseTickers******'
def parseTickers(begin=600000, end=603366):
nonExistentTickers = findAllNonExistentTickers()
existingTickers = findAllExistentTickers()
print nonExistentTickers
print existingTickers
#parse shanghai tickers
for code in range(begin, end + 1):
if (str(code) in nonExistentTickers):
print 'Non-existent ticker***' + str(code)
continue
elif (str(code) in existingTickers):
print 'Existing ticker***' + str(code)
continue
code2 = str(code) + '.SS'
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys'
print url
page = parse(url).getroot()
if page is None:
print "Fail to query for " + str(code)
return
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
stock = Stock(str(code))
if (errorMsg is None):
stock.name = ''
high = page.xpath('//dayshigh')[0].text
if (high is None):
print 'Non-existent ticker***' + str(code)
saveNonExistentTicker(stock)
continue
stock.high = float(high)
stock.low = float(page.xpath('//dayslow')[0].text)
stock.yearHigh = float(page.xpath('//yearhigh')[0].text)
stock.yearLow = float(page.xpath('//yearlow')[0].text)
close = page.xpath('//bid')[0].text
# print close
# print type(close)
if close is not None:
stock.close = float(close)
if (stock.high >= stock.yearHigh):
print 'stock trigger new high index*****' + code2
with io.open('nh.xml', 'wb') as f:
f.writelines(code2)
elif (stock.low <= stock.yearLow):
print 'stock trigger new low index*****' + code2
with io.open('nl.xml', 'wb') as f:
f.writelines(code2)
#print stock
saveTicker(stock)
#parse key statistics data from reuters
from reutersparser import parseKeyStatData
parseKeyStatData(code)
else:
saveNonExistentTicker(stock)
print 'Non-existent ticker***' + str(code)
print 'Finish parseTickers******'
def findQuoteByCode(code,condition=settings.HIGHER):
historyDatas = db.tickers
stock = historyDatas.find_one({"code":code});
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
if code.startswith('6'):
s.isInSh = True;
return s
def findQuoteByCode(code, condition=settings.HIGHER):
historyDatas = db.tickers
stock = historyDatas.find_one({"code": code})
code = stock.get('code')
s = Stock(code)
if condition == settings.HIGHER:
s.PercentChangeFromYearLow = stock.get('percentFromYearLow')
else:
s.PercentChangeFromYearHigh = stock.get('percentFromYearHigh')
s.yearHigh = stock.get('yearHigh')
s.yearLow = stock.get('yearLow')
s.current = stock.get('current')
s.ma50 = stock.get('ma50')
s.ma200 = stock.get('ma200')
if code.startswith('6'):
s.isInSh = True
return s
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code +".SS"
else:
code2 = code +".SZ"
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22'+code2+'%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&diagnostics=true'
logger.debug(url)
try:
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid');
errorMsg = r[0].text
if (errorMsg is None):
print 'OK'
stock = Stock(code)
stock.name = ''
else:
print 'error'
quote = page.xpath('/html/body/query/results/quote');
#print len(quote)
#print len(page.xpath('//ask'))
#print page.xpath('/html/body/query/results/quote/ask[1]/text()')
#print page.xpath('//ask[1]/text()')[0]#both works
yearLow = page.xpath('//yearlow[1]/text()')[0]
logger.debug('yearLow'+yearLow)
yearHigh = page.xpath('//yearhigh[1]/text()')[0]
print 'yearHigh'+yearHigh
PercentChangeFromYearLow = page.xpath('//percentchangefromyearlow[1]/text()')[0]
print 'PercentChangeFromYearLow'+PercentChangeFromYearLow
PercebtChangeFromYearHigh = page.xpath('//percebtchangefromyearhigh[1]/text()')[0]
print 'PercebtChangeFromYearHigh'+PercebtChangeFromYearHigh
FiftydayMovingAverage = page.xpath('//fiftydaymovingaverage[1]/text()')[0]
print 'FiftydayMovingAverage'+FiftydayMovingAverage
TwoHundreddayMovingAverage = page.xpath('//twohundreddaymovingaverage[1]/text()')[0]
print 'TwoHundreddayMovingAverage'+TwoHundreddayMovingAverage
PercentChangeFromTwoHundreddayMovingAverage = page.xpath('//percentchangefromtwohundreddaymovingaverage[1]/text()')[0]
print 'PercentChangeFromTwoHundreddayMovingAverage'+PercentChangeFromTwoHundreddayMovingAverage
PercentChangeFromFiftydayMovingAverage = page.xpath('//percentchangefromfiftydaymovingaverage[1]/text()')[0]
print 'PercentChangeFromFiftydayMovingAverage'+PercentChangeFromFiftydayMovingAverage
logger.debug(page.xpath('//DaysLow[1]/text()'))
LastTradePriceOnly = page.xpath('//lasttradepriceonly[1]/text()')[0]
stock = Stock(code)
stock.yearHigh = float(yearHigh)
stock.yearLow = float(yearLow)
#stock.PercebtChangeFromYearHigh = float(PercebtChangeFromYearHigh.rstrip('%'))
stock.PercentChangeFromYearHigh = float(PercebtChangeFromYearHigh.lstrip('-').rstrip('%'))
stock.PercentChangeFromYearLow = float(PercentChangeFromYearLow.lstrip('+').rstrip('%'))
stock.ma50 = float(FiftydayMovingAverage)
stock.ma200 = float(TwoHundreddayMovingAverage)
if LastTradePriceOnly is not None:
logger.debug("{} current: {}".format(code,LastTradePriceOnly))
stock.current = float(LastTradePriceOnly)
close = page.xpath('//bid')[0].text;
if close is not None:
stock.close = float(close)
return stock
except Exception as ex:
logger.error('Fail to download latest update from yahoo API:'+code)
traceback.print_exc(file=sys.stdout)
logger.exception(ex)
return None
def parseFinanceData(code):
from lxml import etree
from lxml.html import parse
if (len(code) == 9):
code2 = code
elif (code.startswith('6')):
code2 = code + ".SS"
else:
code2 = code + ".SZ"
url = 'http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(%22' + code2 + '%22)&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&diagnostics=true'
logger.debug(url)
try:
page = parse(url).getroot()
result = etree.tostring(page)
print result
r = page.xpath('//errorindicationreturnedforsymbolchangedinvalid')
errorMsg = r[0].text
if (errorMsg is None):
print 'OK'
stock = Stock(code)
stock.name = ''
else:
print 'error'
quote = page.xpath('/html/body/query/results/quote')
#print len(quote)
#print len(page.xpath('//ask'))
#print page.xpath('/html/body/query/results/quote/ask[1]/text()')
#print page.xpath('//ask[1]/text()')[0]#both works
yearLow = page.xpath('//yearlow[1]/text()')[0]
logger.debug('yearLow' + yearLow)
yearHigh = page.xpath('//yearhigh[1]/text()')[0]
print 'yearHigh' + yearHigh
PercentChangeFromYearLow = page.xpath(
'//percentchangefromyearlow[1]/text()')[0]
print 'PercentChangeFromYearLow' + PercentChangeFromYearLow
PercebtChangeFromYearHigh = page.xpath(
'//percebtchangefromyearhigh[1]/text()')[0]
print 'PercebtChangeFromYearHigh' + PercebtChangeFromYearHigh
FiftydayMovingAverage = page.xpath(
'//fiftydaymovingaverage[1]/text()')[0]
print 'FiftydayMovingAverage' + FiftydayMovingAverage
TwoHundreddayMovingAverage = page.xpath(
'//twohundreddaymovingaverage[1]/text()')[0]
print 'TwoHundreddayMovingAverage' + TwoHundreddayMovingAverage
PercentChangeFromTwoHundreddayMovingAverage = page.xpath(
'//percentchangefromtwohundreddaymovingaverage[1]/text()')[0]
print 'PercentChangeFromTwoHundreddayMovingAverage' + PercentChangeFromTwoHundreddayMovingAverage
PercentChangeFromFiftydayMovingAverage = page.xpath(
'//percentchangefromfiftydaymovingaverage[1]/text()')[0]
print 'PercentChangeFromFiftydayMovingAverage' + PercentChangeFromFiftydayMovingAverage
logger.debug(page.xpath('//DaysLow[1]/text()'))
LastTradePriceOnly = page.xpath('//lasttradepriceonly[1]/text()')[0]
stock = Stock(code)
stock.yearHigh = float(yearHigh)
stock.yearLow = float(yearLow)
#stock.PercebtChangeFromYearHigh = float(PercebtChangeFromYearHigh.rstrip('%'))
stock.PercentChangeFromYearHigh = float(
PercebtChangeFromYearHigh.lstrip('-').rstrip('%'))
stock.PercentChangeFromYearLow = float(
PercentChangeFromYearLow.lstrip('+').rstrip('%'))
stock.ma50 = float(FiftydayMovingAverage)
stock.ma200 = float(TwoHundreddayMovingAverage)
if LastTradePriceOnly is not None:
logger.debug("{} current: {}".format(code, LastTradePriceOnly))
stock.current = float(LastTradePriceOnly)
close = page.xpath('//bid')[0].text
if close is not None:
stock.close = float(close)
return stock
except Exception as ex:
logger.error('Fail to download latest update from yahoo API:' + code)
traceback.print_exc(file=sys.stdout)
logger.exception(ex)
return None
