def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
globals.data[threadName + '_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while (globals.data[threadName + '_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
downloadErrorCnt += 1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(
tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page + 1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def run(subcmd, params):
if (subcmd == 'stockhisdata'):
return downloadStockHisData()
else:
utils.output('未知命令', 'red')
return
def showStockChart(stockcode):
stockcode = stock.normalizeStockCode(stockcode)
kdata = stock.getStockDayDataQianFuQuan(stockcode)
if (not kdata):
utils.output('股票数据不存在', 'red')
return False
if (stockcode[:3] != 'sh0'):
weightdata = stock.getStockWeightData(stockcode)
if (not weightdata):
utils.output('复权数据不存在', 'red')
return False
#print(weightdata)
#set data
globals.mainwin.chart.kdata = kdata
globals.mainwin.chart.stockcode = stockcode
globals.mainwin.scrollbar.setMinimum(0)
ktotalcount = len(kdata)
globals.mainwin.scrollbar.setMaximum(ktotalcount)
globals.mainwin.scrollbar.setValue(ktotalcount)
globals.mainwin.scrollbar.setSingleStep(1)
if (stockcode[:3] != 'sh0'):
globals.mainwin.chart.weightdata = weightdata
#draw
globals.viewportclear = False
utils.update()
def cancelDownload(threadName):
def stopThread(threadName):
threadList = []
try:
threadList = globals.data[threadName+'_threadlist']
globals.data[threadName+'_threadlist'] = None
globals.data[threadName+'_running'] = False
except:
pass
if(threadList):
for t in threadList:
t.stop()
threadLeft = 0
stopThread(threadName)
for t in threading.enumerate():
if(t.getName() == threadName):
threadLeft +=1
if(threadLeft > 0):
tipstr = '终止下载中, 目前剩余'
outstr = '{} {} 线程'.format(tipstr, threadLeft)
utils.overwrite(outstr, tipstr)
return False
else:
utils.output('所有下载线程已终止')
return True
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
globals.data[threadName+'_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while(globals.data[threadName+'_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
downloadErrorCnt +=1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page +1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def showStockChart(stockcode):
stockcode = stock.normalizeStockCode(stockcode)
kdata = stock.getStockDayDataQianFuQuan(stockcode)
if(not kdata):
utils.output('股票数据不存在', 'red')
return False
if(stockcode[:3] != 'sh0'):
weightdata = stock.getStockWeightData(stockcode)
if(not weightdata):
utils.output('复权数据不存在', 'red')
return False
#print(weightdata)
#set data
globals.mainwin.chart.kdata = kdata
globals.mainwin.chart.stockcode = stockcode
globals.mainwin.scrollbar.setMinimum(0)
ktotalcount = len(kdata)
globals.mainwin.scrollbar.setMaximum(ktotalcount)
globals.mainwin.scrollbar.setValue(ktotalcount)
globals.mainwin.scrollbar.setSingleStep(1)
if(stockcode[:3] != 'sh0'):
globals.mainwin.chart.weightdata = weightdata
#draw
globals.viewportclear = False
utils.update()
def getHq(stock_code):
url = 'http://hq.sinajs.cn/rn={}&list={}'.format(
random.randint(1, 1000000), stock_code)
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
dataArr = re.findall(r'"(.*)"', content)[0].split(',')
if (len(dataArr) < 31):
utils.output('no data')
return False
info = {}
info['name'] = dataArr[0]
info['code'] = stock_code
info['open'] = float(dataArr[1])
info['close_yesterday'] = float(dataArr[2])
info['price'] = float(dataArr[3])
info['close'] = info['price']
info['high'] = float(dataArr[4])
info['low'] = float(dataArr[5])
info['amount'] = float(dataArr[9]) / 1000
info['date2'] = dataArr[30]
info['date'] = info['date2'].replace('-', '')
info['time'] = dataArr[31]
if info['close_yesterday'] > 0:
info['price_chg'] = (info['price'] - info['close_yesterday']
) * 100 / info['close_yesterday']
if (info['open'] == 0):
info['open'] = info['price']
return info
def run(subcmd, params):
if (subcmd == 'code'):
if (len(params) != 1):
utils.output('参数错误', 'red')
utils.output(subCommands['code'])
return
return _suggestCode(params[0])
def run(subcmd, params):
if(subcmd == 'stockhisdata'):
return downloadStockHisData()
else:
utils.output('未知命令', 'red')
return
def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if(subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt +=1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if(didata[-1] and didata[-1].get('tip')):
filterResults.append({'code':stockcode, 'date':didata[-1]['date']})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row=[]
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def _getHq(stock_code):
info = stock.getHq(stock_code)
if(info):
str = '{} {} {} {:-.2f}%'.format(info['code'], info['name'], info['price'], info['price_chg'])
color = ''
if info['price_chg'] >0 :
color = 'red'
elif info['price_chg'] < 0 :
color = 'green'
utils.output(str, color)
def run(subcmd, params):
if(subcmd == 'show'):
if(len(params) == 1):
return showStockChart(params[0])
else:
utils.output('请给出股票代码', 'red')
utils.output(subCommands['go'])
return
elif(subcmd == 'clear'):
globals.viewportclear = True
utils.update()
def _getHq(stock_code):
info = stock.getHq(stock_code)
if (info):
str = '{} {} {} {:-.2f}%'.format(info['code'], info['name'],
info['price'], info['price_chg'])
color = ''
if info['price_chg'] > 0:
color = 'red'
elif info['price_chg'] < 0:
color = 'green'
utils.output(str, color)
def run(subcmd, params):
if (subcmd == 'show'):
if (len(params) == 1):
return showStockChart(params[0])
else:
utils.output('请给出股票代码', 'red')
utils.output(subCommands['go'])
return
elif (subcmd == 'clear'):
globals.viewportclear = True
utils.update()
def _getShanghaiACount():
url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a'
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
cnt = re.findall(r'"([0-9]+)"', content)
utils.output(cnt)
def query(self, sql, parameters=None):
try:
if (parameters):
self.cursor.execute(sql, parameters)
else:
self.cursor.execute(sql)
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db query error: {}'.format(error), 'red')
return False
return True
def query(self, sql, parameters = None):
try:
if(parameters):
self.cursor.execute(sql, parameters)
else:
self.cursor.execute(sql)
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db query error: {}'.format(error), 'red')
return False
return True
def _getShanghaiACount():
url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a'
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
cnt = re.findall(r'"([0-9]+)"', content)
utils.output(cnt)
def run(subcmd, params):
if(subcmd == 'stockhisdata'):
while(True):
if(cancelDownload('download_stockhisdata')):
break
time.sleep(0.01)
QCoreApplication.processEvents()
return
else:
utils.output('未知命令', 'red')
return
def getStockList():
stocklistfile = globals.datapath + '/stocklist.txt'
stocklistdata = utils.file_get_contents(stocklistfile)
if (not stocklistdata):
utils.output('股票列表载入失败, 请先运行update stocklist', 'red')
return False
stocklist = {}
lines = stocklistdata.split("\n")
for line in lines:
if (line):
stockArr = line.split(',')
if (len(stockArr) == 2):
scode = normalizeStockCode(stockArr[0])
stocklist[scode] = stockArr[1]
return stocklist
def suggestSubcmd(self, cmd, subcmd):
filepath = './commands/' + cmd + '.py'
if not os.path.exists(filepath):
return
modname = 'commands.{}'.format(cmd)
__import__(modname)
module = sys.modules[modname]
if (globals.debug):
imp.reload(module)
if (not subcmd): #no subcommand
if (len(module.subCommands.keys()) == 1
): #only have one subcommand
subcmd = list(module.subCommands.keys())[0]
cmdLine = cmd + ' ' + subcmd + ' '
self.setText(cmdLine)
return
elif (len(module.subCommands) > 1):
utils.output(list(module.subCommands.keys()))
subcmd = self._suggest('', list(module.subCommands.keys()))
cmdLine = cmd + ' ' + subcmd
self.setText(cmdLine)
return
#have subcommand
suggestSubcmd = []
for sc in module.subCommands.keys():
if (subcmd and sc[:len(subcmd)] == subcmd):
suggestSubcmd.append(sc)
if (len(suggestSubcmd) == 1): #only find one subcommand
subcmd = suggestSubcmd[0]
cmdLine = cmd + ' ' + subcmd + ' '
self.setText(cmdLine)
return
elif (len(suggestSubcmd) > 1):
subcmd = self._suggest(subcmd, suggestSubcmd)
cmdLine = cmd + ' ' + subcmd
self.setText(cmdLine)
utils.output(suggestSubcmd)
return
def suggestSubcmd(self, cmd, subcmd):
filepath = './commands/'+cmd+'.py'
if not os.path.exists(filepath):
return
modname = 'commands.{}'.format(cmd)
__import__(modname)
module = sys.modules[modname]
if(globals.debug):
imp.reload(module)
if(not subcmd): #no subcommand
if(len(module.subCommands.keys()) == 1): #only have one subcommand
subcmd = list(module.subCommands.keys())[0]
cmdLine = cmd + ' ' + subcmd + ' '
self.setText(cmdLine)
return
elif(len(module.subCommands) > 1):
utils.output(list(module.subCommands.keys()))
subcmd = self._suggest('', list(module.subCommands.keys()))
cmdLine = cmd + ' ' + subcmd
self.setText(cmdLine)
return
#have subcommand
suggestSubcmd = []
for sc in module.subCommands.keys():
if(subcmd and sc[:len(subcmd)] == subcmd):
suggestSubcmd.append(sc)
if(len(suggestSubcmd) == 1): #only find one subcommand
subcmd = suggestSubcmd[0]
cmdLine = cmd + ' ' + subcmd + ' '
self.setText(cmdLine)
return
elif(len(suggestSubcmd) > 1):
subcmd = self._suggest(subcmd, suggestSubcmd)
cmdLine = cmd + ' ' + subcmd
self.setText(cmdLine)
utils.output(suggestSubcmd)
return
def selectAll(self, sql, parameters = None):
rows = []
c = self.conn.cursor()
try:
if(parameters):
c.execute(sql, parameters)
else:
c.execute(sql)
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db selectAll error: {}'.format(error), 'red')
return False
for r in c:
rows.append(r)
c.close()
return rows
def selectAll(self, sql, parameters=None):
rows = []
c = self.conn.cursor()
try:
if (parameters):
c.execute(sql, parameters)
else:
c.execute(sql)
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db selectAll error: {}'.format(error), 'red')
return False
for r in c:
rows.append(r)
c.close()
return rows
def suggestCmd(self, cmdLine):
cmds = cmdLine.split(' ')
cmd = cmds[0]
try:
subcmd = cmds[1]
except IndexError:
subcmd = None
param = None
if (subcmd):
try:
param = cmds[2]
except IndexError:
pass
if (cmd and (not subcmd)): #only have command
#try to suggest command
suggestArr = []
for c in utils.getCmdList():
if (c[:len(cmd)] == cmd):
suggestArr.append(c)
if (len(suggestArr) == 1
): #now we got the only one correct command
cmd = suggestArr[0]
cmdLine = cmd + ' '
if (self.text() == cmdLine): #need suggest subcommand
self.suggestSubcmd(cmd, None)
else:
self.setText(cmdLine)
return
elif (len(suggestArr) > 1): #output all possible cmds
cmd = self._suggest(cmd, suggestArr)
cmdLine = cmd
self.setText(cmdLine)
utils.output(suggestArr)
return
#now we have command and subcommand
if (subcmd and (not param)):
self.suggestSubcmd(cmd, subcmd)
def connect(self):
try:
self.conn = sqlite3.connect(self.dbfile)
self.conn.row_factory = dict_factory
self.cursor = self.conn.cursor()
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db connect error: {}'.format(error), 'red')
return False
sqlList = []
sqlList.append('PRAGMA cache_size =8000')
sqlList.append('PRAGMA synchronous = 0')
sqlList.append('PRAGMA encoding = "UTF-8"')
for sql in sqlList:
self.query(sql)
self.commit()
return self.conn
def suggestCmd(self, cmdLine):
cmds = cmdLine.split(' ')
cmd = cmds[0]
try:
subcmd = cmds[1]
except IndexError:
subcmd = None
param = None
if(subcmd):
try:
param = cmds[2]
except IndexError:
pass
if(cmd and (not subcmd)): #only have command
#try to suggest command
suggestArr = []
for c in utils.getCmdList():
if(c[:len(cmd)] == cmd):
suggestArr.append(c)
if(len(suggestArr) == 1): #now we got the only one correct command
cmd = suggestArr[0]
cmdLine = cmd + ' '
if(self.text() == cmdLine): #need suggest subcommand
self.suggestSubcmd(cmd, None)
else:
self.setText(cmdLine)
return
elif(len(suggestArr) > 1): #output all possible cmds
cmd = self._suggest(cmd, suggestArr)
cmdLine = cmd
self.setText(cmdLine)
utils.output(suggestArr)
return
#now we have command and subcommand
if(subcmd and (not param)):
self.suggestSubcmd(cmd, subcmd)
def run(subcmd, params):
if(subcmd == 'list'):
setting = {}
for k in subCommands.keys():
if(k != 'list'):
setting[k] = globals.__dict__.get(k)
utils.output(setting)
elif(subcmd == 'realtime'):
if(len(params) < 1):
msg = 'realtime on'
globals.realtime = True
utils.output(msg)
elif(len(params) == 1 and (params[0]=='on' or params[0]=='off')):
if(params[0]=='on'):
msg = 'realtime on'
globals.realtime = True
else:
msg = 'realtime off'
globals.realtime = False
utils.output(msg)
else:
utils.output('参数错误', 'red')
utils.output(subCommands['realtime'])
if(not globals.realtime):
chart = globals.mainwin.chart
hqinfo = chart.rt_data.get('data')
if(hqinfo and chart.rt_data.get('status') == 'realtime'):
chart.rt_data['status'] = ''
if(chart.kdata[-1]['date'] == hqinfo['date']):
chart.kdata = chart.kdata[:-1]
globals.mainwin.scrollbar.setMaximum(len(chart.kdata))
utils.update()
def connect(self):
try:
self.conn = sqlite3.connect(self.dbfile)
self.conn.row_factory = dict_factory
self.cursor = self.conn.cursor()
except:
exc_type, exc_value = sys.exc_info()[:2]
error = '{}: {}'.format(exc_type.__name__, exc_value)
utils.output('db connect error: {}'.format(error), 'red')
return False
sqlList = []
sqlList.append('PRAGMA cache_size =8000')
sqlList.append('PRAGMA synchronous = 0')
sqlList.append('PRAGMA encoding = "UTF-8"')
for sql in sqlList:
self.query(sql)
self.commit()
return self.conn
def run(subcmd, params):
if (subcmd == 'shanghai_a_count'):
return _getShanghaiACount()
if (subcmd == 'hq'):
if (len(params) != 1):
utils.output('请给出股票代码', 'red')
utils.output(subCommands['hq'])
return
stock_code = stock.normalizeStockCode(params[0])
if (not stock_code):
utils.output('股票代码错误', 'red')
utils.output(subCommands['hq'])
return
return _getHq(stock_code)
def run(subcmd, params):
if(subcmd == 'shanghai_a_count'):
return _getShanghaiACount()
if(subcmd == 'hq'):
if(len(params) != 1):
utils.output('请给出股票代码', 'red')
utils.output(subCommands['hq'])
return
stock_code = stock.normalizeStockCode(params[0])
if(not stock_code):
utils.output('股票代码错误', 'red')
utils.output(subCommands['hq'])
return
return _getHq(stock_code)
def _suggestCode(text):
escapeTxt = urllib.parse.quote(text)
url = 'http://suggest3.sinajs.cn/suggest/type=&key={}'.format(escapeTxt)
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return 'eee'
content = content.decode('gbk')
codeArr = []
resultArr = re.findall(r'"(.*)"', content)[0].split(';')
for line in resultArr:
cArr = line.split(',')
if (len(cArr) == 6 and cArr[1] == '11'): #cArr[1]==11 A股
codestr = '{} {}'.format(cArr[3], cArr[4])
codeArr.append(codestr)
if (len(codeArr) < 1):
utils.output('nothing')
else:
utils.output(codeArr)
return
def downloadStockHisData():
threadName = 'download_stockhisdata'
globals.data[threadName+'_threadlist'] = None
globals.data[threadName+'_running'] = True
threadCount = 10
start_time = datetime.datetime.now()
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
stocklist = stock.getStockList()
if((not stocklist) or len(stocklist)<10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#build fetch list
in_queue = queue.Queue()
stockcodes = list(stocklist.keys())
stockcodes.sort()
for scode in stockcodes:
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if(scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
#check diskcache before put to download queue
in_queue.put(startUrl)
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
globals.data[threadName+'_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while(globals.data[threadName+'_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
downloadErrorCnt +=1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page +1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
downloadErrorCnt = downloadData(in_queue)
if(not globals.data[threadName+'_running']):
utils.output('下载已被终止', 'red')
return
if(downloadErrorCnt != 0):
utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt))
return
utils.output('下载完成')
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1))
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if (len(totalpagelist) < 1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if (len(totalstocklist) < 1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage + 1):
url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while (page <= totalpage):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page,
totalpage)
if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page + 1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if (not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if (len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist),
totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if (not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if((not stocklist) or len(stocklist)<10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if(scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if(err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt +=1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S)
if(len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S)
if(len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S)
if(len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist=[]
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if(not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, ))
if(row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if(len(hisDataResults)< 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2])
if(stype == 'stock' and len(sdata)!=7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif(stype == 'index' and len(sdata)!=6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if(stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def run(subcmd, params):
utils.output(threading.enumerate())
def run(subcmd, params):
utils.output(threading.enumerate())
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page-1)*50+1))
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if(len(totalpagelist)<1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if(len(totalstocklist)<1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage+1):
url = '{}{}'.format(partUrl, ((i-1)*50+1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while(page <= totalpage):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage)
if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page +1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if(not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if(len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if(not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def run(subcmd, params):
cmdList = utils.getCmdList()
utils.output(desc)
utils.output(cmdList)
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if ((not stocklist) or len(stocklist) < 10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile(
"<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>",
re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if (scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt += 1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->',
content, re.S)
if (len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>',
stocksection[0], re.S)
if (len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>',
stockyearsecton[0], re.S)
if (len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist = []
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if (not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne(
'select * from stockhistory where code =? order by date desc limit 1',
(scode, ))
if (row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if (len(hisDataResults) < 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if (getYearAndJidu(lastupdate) == getYearAndJidu(
datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>',
res[2])
if (stype == 'stock' and len(sdata) != 7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif (stype == 'index' and len(sdata) != 6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if (stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query(
'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume,
qamount, qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def keyPressEvent(self, event):
super().keyPressEvent(event)
key = event.key()
if (key == Qt.Key_Escape):
globals.mainwin.chart.setFocus(Qt.OtherFocusReason)
globals.mainwin.removeDockWidget(globals.mainwin.dockOne)
elif (key == Qt.Key_Return):
res = re.findall(r'(\S+)', self.text())
text = ' '.join(res)
if (text != ''):
#add command history
if (not self.cmdHistory
or self.cmdHistory[len(self.cmdHistory) - 1] != text):
self.cmdHistory.append(text)
length = len(self.cmdHistory)
if (length > 100):
self.cmdHistory = self.cmdHistory[length - 50:]
self.cmdHistoryIdx = len(self.cmdHistory)
#run command
utils.output(text, 'blue')
self.clear()
utils.runCmd(text)
elif (key == Qt.Key_U):
if (event.modifiers() == Qt.ControlModifier):
pos = self.cursorPosition()
txt = self.text()[pos:]
self.setText(txt)
self.setCursorPosition(0)
elif (key == Qt.Key_K):
if (event.modifiers() == Qt.ControlModifier):
pos = self.cursorPosition()
txt = self.text()[:pos]
self.setText(txt)
elif (key == Qt.Key_Up):
idx = self.cmdHistoryIdx - 1
if (idx < 0):
idx = len(self.cmdHistory) - 1
try:
self.setText(self.cmdHistory[idx])
self.cmdHistoryIdx = idx
except:
pass
elif (key == Qt.Key_Down):
idx = self.cmdHistoryIdx + 1
if (idx > len(self.cmdHistory) - 1):
idx = 0
try:
self.setText(self.cmdHistory[idx])
self.cmdHistoryIdx = idx
except:
pass
elif (key == Qt.Key_Tab):
res = re.findall(r'(\S+)', self.text())
text = ' '.join(res)
if (text != ''):
self.suggestCmd(text)
def downloadStockHisData():
threadName = 'download_stockhisdata'
globals.data[threadName + '_threadlist'] = None
globals.data[threadName + '_running'] = True
threadCount = 10
start_time = datetime.datetime.now()
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
stocklist = stock.getStockList()
if ((not stocklist) or len(stocklist) < 10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#build fetch list
in_queue = queue.Queue()
stockcodes = list(stocklist.keys())
stockcodes.sort()
for scode in stockcodes:
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if (scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
#check diskcache before put to download queue
in_queue.put(startUrl)
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
globals.data[threadName + '_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while (globals.data[threadName + '_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
downloadErrorCnt += 1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(
tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page + 1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
downloadErrorCnt = downloadData(in_queue)
if (not globals.data[threadName + '_running']):
utils.output('下载已被终止', 'red')
return
if (downloadErrorCnt != 0):
utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt))
return
utils.output('下载完成')
def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if (subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt += 1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(
prefixstr, cnt, totalstock, stockcode, len(filterResults),
diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if (didata[-1] and didata[-1].get('tip')):
filterResults.append({
'code': stockcode,
'date': didata[-1]['date']
})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row = []
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def keyPressEvent(self, event):
super().keyPressEvent(event)
key = event.key()
if(key == Qt.Key_Escape):
globals.mainwin.chart.setFocus(Qt.OtherFocusReason)
globals.mainwin.removeDockWidget(globals.mainwin.dockOne)
elif(key == Qt.Key_Return):
res = re.findall(r'(\S+)', self.text())
text = ' '.join(res)
if(text!=''):
#add command history
if(not self.cmdHistory or self.cmdHistory[len(self.cmdHistory)-1] != text):
self.cmdHistory.append(text)
length = len(self.cmdHistory)
if(length>100):
self.cmdHistory = self.cmdHistory[length-50:]
self.cmdHistoryIdx = len(self.cmdHistory)
#run command
utils.output(text,'blue')
self.clear()
utils.runCmd(text)
elif(key == Qt.Key_U):
if(event.modifiers() == Qt.ControlModifier):
pos = self.cursorPosition()
txt = self.text()[pos:]
self.setText(txt)
self.setCursorPosition(0)
elif(key == Qt.Key_K):
if(event.modifiers() == Qt.ControlModifier):
pos = self.cursorPosition()
txt = self.text()[:pos]
self.setText(txt)
elif(key == Qt.Key_Up):
idx = self.cmdHistoryIdx - 1
if(idx < 0):
idx = len(self.cmdHistory) - 1
try:
self.setText(self.cmdHistory[idx])
self.cmdHistoryIdx = idx
except:
pass
elif(key == Qt.Key_Down):
idx = self.cmdHistoryIdx + 1
if(idx > len(self.cmdHistory) -1):
idx = 0
try:
self.setText(self.cmdHistory[idx])
self.cmdHistoryIdx = idx
except:
pass
elif(key == Qt.Key_Tab):
res = re.findall(r'(\S+)', self.text())
text = ' '.join(res)
if(text!=''):
self.suggestCmd(text)
