def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName + '_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while (globals.data[threadName + '_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if (err): utils.output(err, 'red') downloadErrorCnt += 1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format( tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page + 1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt
def run(subcmd, params): if (subcmd == 'stockhisdata'): return downloadStockHisData() else: utils.output('未知命令', 'red') return
def showStockChart(stockcode): stockcode = stock.normalizeStockCode(stockcode) kdata = stock.getStockDayDataQianFuQuan(stockcode) if (not kdata): utils.output('股票数据不存在', 'red') return False if (stockcode[:3] != 'sh0'): weightdata = stock.getStockWeightData(stockcode) if (not weightdata): utils.output('复权数据不存在', 'red') return False #print(weightdata) #set data globals.mainwin.chart.kdata = kdata globals.mainwin.chart.stockcode = stockcode globals.mainwin.scrollbar.setMinimum(0) ktotalcount = len(kdata) globals.mainwin.scrollbar.setMaximum(ktotalcount) globals.mainwin.scrollbar.setValue(ktotalcount) globals.mainwin.scrollbar.setSingleStep(1) if (stockcode[:3] != 'sh0'): globals.mainwin.chart.weightdata = weightdata #draw globals.viewportclear = False utils.update()
def cancelDownload(threadName): def stopThread(threadName): threadList = [] try: threadList = globals.data[threadName+'_threadlist'] globals.data[threadName+'_threadlist'] = None globals.data[threadName+'_running'] = False except: pass if(threadList): for t in threadList: t.stop() threadLeft = 0 stopThread(threadName) for t in threading.enumerate(): if(t.getName() == threadName): threadLeft +=1 if(threadLeft > 0): tipstr = '终止下载中, 目前剩余' outstr = '{} {} 线程'.format(tipstr, threadLeft) utils.overwrite(outstr, tipstr) return False else: utils.output('所有下载线程已终止') return True
def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName+'_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while(globals.data[threadName+'_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if(err): utils.output(err, 'red') downloadErrorCnt +=1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page +1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt
def showStockChart(stockcode): stockcode = stock.normalizeStockCode(stockcode) kdata = stock.getStockDayDataQianFuQuan(stockcode) if(not kdata): utils.output('股票数据不存在', 'red') return False if(stockcode[:3] != 'sh0'): weightdata = stock.getStockWeightData(stockcode) if(not weightdata): utils.output('复权数据不存在', 'red') return False #print(weightdata) #set data globals.mainwin.chart.kdata = kdata globals.mainwin.chart.stockcode = stockcode globals.mainwin.scrollbar.setMinimum(0) ktotalcount = len(kdata) globals.mainwin.scrollbar.setMaximum(ktotalcount) globals.mainwin.scrollbar.setValue(ktotalcount) globals.mainwin.scrollbar.setSingleStep(1) if(stockcode[:3] != 'sh0'): globals.mainwin.chart.weightdata = weightdata #draw globals.viewportclear = False utils.update()
def getHq(stock_code): url = 'http://hq.sinajs.cn/rn={}&list={}'.format( random.randint(1, 1000000), stock_code) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False content = content.decode('gbk') dataArr = re.findall(r'"(.*)"', content)[0].split(',') if (len(dataArr) < 31): utils.output('no data') return False info = {} info['name'] = dataArr[0] info['code'] = stock_code info['open'] = float(dataArr[1]) info['close_yesterday'] = float(dataArr[2]) info['price'] = float(dataArr[3]) info['close'] = info['price'] info['high'] = float(dataArr[4]) info['low'] = float(dataArr[5]) info['amount'] = float(dataArr[9]) / 1000 info['date2'] = dataArr[30] info['date'] = info['date2'].replace('-', '') info['time'] = dataArr[31] if info['close_yesterday'] > 0: info['price_chg'] = (info['price'] - info['close_yesterday'] ) * 100 / info['close_yesterday'] if (info['open'] == 0): info['open'] = info['price'] return info
def run(subcmd, params): if (subcmd == 'code'): if (len(params) != 1): utils.output('参数错误', 'red') utils.output(subCommands['code']) return return _suggestCode(params[0])
def run(subcmd, params): if(subcmd == 'stockhisdata'): return downloadStockHisData() else: utils.output('未知命令', 'red') return
def run(subcmd, params): chart = globals.mainwin.chart stocklist = stock.getStockList() if(subcmd == 'di'): filterResults = [] di = indicators.di.Di() utils.output('开始di条件过滤') prefixstr = 'filter di条件过滤:' totalstock = len(stocklist) cnt = 0 start_time = datetime.datetime.now() for stockcode in stocklist: cnt +=1 stockcode = stock.normalizeStockCode(stockcode) end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr) utils.overwrite(outstr, prefixstr) stockdata = stock.getStockDayDataQianFuQuan(stockcode) stockdata = stockdata[-500:] didata = di.calculateData(stockdata) QCoreApplication.processEvents() if(didata[-1] and didata[-1].get('tip')): filterResults.append({'code':stockcode, 'date':didata[-1]['date']}) dataArr = [] for row in filterResults: scode = row['code'] sdate = row['date'] row=[] row.append(scode) row.append(stock.getStockName(scode)) row.append(sdate) dataArr.append(row) #write to file begin filename = globals.datapath + '/lastFilterResult.dat' try: f = open(filename, 'wb') except IOError: exc_type, exc_value = sys.exc_info()[:2] errmsg = '{}: {}'.format(exc_type.__name__, exc_value) output(errmsg, 'red') return False pickle.dump(dataArr, f) f.close() #write to file end tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None) tbResults = globals.mainwin.stockfilterresultdlg.tbResults tbResults.setModel(tablemodel) globals.mainwin.windowStockFilterResult()
def _getHq(stock_code): info = stock.getHq(stock_code) if(info): str = '{} {} {} {:-.2f}%'.format(info['code'], info['name'], info['price'], info['price_chg']) color = '' if info['price_chg'] >0 : color = 'red' elif info['price_chg'] < 0 : color = 'green' utils.output(str, color)
def run(subcmd, params): if(subcmd == 'show'): if(len(params) == 1): return showStockChart(params[0]) else: utils.output('请给出股票代码', 'red') utils.output(subCommands['go']) return elif(subcmd == 'clear'): globals.viewportclear = True utils.update()
def _getHq(stock_code): info = stock.getHq(stock_code) if (info): str = '{} {} {} {:-.2f}%'.format(info['code'], info['name'], info['price'], info['price_chg']) color = '' if info['price_chg'] > 0: color = 'red' elif info['price_chg'] < 0: color = 'green' utils.output(str, color)
def run(subcmd, params): if (subcmd == 'show'): if (len(params) == 1): return showStockChart(params[0]) else: utils.output('请给出股票代码', 'red') utils.output(subCommands['go']) return elif (subcmd == 'clear'): globals.viewportclear = True utils.update()
def _getShanghaiACount(): url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a' err, u, content = utils.getWebContent(url) if(err): utils.output(err, 'red') return False content = content.decode('gbk') cnt = re.findall(r'"([0-9]+)"', content) utils.output(cnt)
def query(self, sql, parameters=None): try: if (parameters): self.cursor.execute(sql, parameters) else: self.cursor.execute(sql) except: exc_type, exc_value = sys.exc_info()[:2] error = '{}: {}'.format(exc_type.__name__, exc_value) utils.output('db query error: {}'.format(error), 'red') return False return True
def query(self, sql, parameters = None): try: if(parameters): self.cursor.execute(sql, parameters) else: self.cursor.execute(sql) except: exc_type, exc_value = sys.exc_info()[:2] error = '{}: {}'.format(exc_type.__name__, exc_value) utils.output('db query error: {}'.format(error), 'red') return False return True
def _getShanghaiACount(): url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a' err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False content = content.decode('gbk') cnt = re.findall(r'"([0-9]+)"', content) utils.output(cnt)
def run(subcmd, params): if(subcmd == 'stockhisdata'): while(True): if(cancelDownload('download_stockhisdata')): break time.sleep(0.01) QCoreApplication.processEvents() return else: utils.output('未知命令', 'red') return
def getStockList(): stocklistfile = globals.datapath + '/stocklist.txt' stocklistdata = utils.file_get_contents(stocklistfile) if (not stocklistdata): utils.output('股票列表载入失败, 请先运行update stocklist', 'red') return False stocklist = {} lines = stocklistdata.split("\n") for line in lines: if (line): stockArr = line.split(',') if (len(stockArr) == 2): scode = normalizeStockCode(stockArr[0]) stocklist[scode] = stockArr[1] return stocklist
def suggestSubcmd(self, cmd, subcmd): filepath = './commands/' + cmd + '.py' if not os.path.exists(filepath): return modname = 'commands.{}'.format(cmd) __import__(modname) module = sys.modules[modname] if (globals.debug): imp.reload(module) if (not subcmd): #no subcommand if (len(module.subCommands.keys()) == 1 ): #only have one subcommand subcmd = list(module.subCommands.keys())[0] cmdLine = cmd + ' ' + subcmd + ' ' self.setText(cmdLine) return elif (len(module.subCommands) > 1): utils.output(list(module.subCommands.keys())) subcmd = self._suggest('', list(module.subCommands.keys())) cmdLine = cmd + ' ' + subcmd self.setText(cmdLine) return #have subcommand suggestSubcmd = [] for sc in module.subCommands.keys(): if (subcmd and sc[:len(subcmd)] == subcmd): suggestSubcmd.append(sc) if (len(suggestSubcmd) == 1): #only find one subcommand subcmd = suggestSubcmd[0] cmdLine = cmd + ' ' + subcmd + ' ' self.setText(cmdLine) return elif (len(suggestSubcmd) > 1): subcmd = self._suggest(subcmd, suggestSubcmd) cmdLine = cmd + ' ' + subcmd self.setText(cmdLine) utils.output(suggestSubcmd) return
def suggestSubcmd(self, cmd, subcmd): filepath = './commands/'+cmd+'.py' if not os.path.exists(filepath): return modname = 'commands.{}'.format(cmd) __import__(modname) module = sys.modules[modname] if(globals.debug): imp.reload(module) if(not subcmd): #no subcommand if(len(module.subCommands.keys()) == 1): #only have one subcommand subcmd = list(module.subCommands.keys())[0] cmdLine = cmd + ' ' + subcmd + ' ' self.setText(cmdLine) return elif(len(module.subCommands) > 1): utils.output(list(module.subCommands.keys())) subcmd = self._suggest('', list(module.subCommands.keys())) cmdLine = cmd + ' ' + subcmd self.setText(cmdLine) return #have subcommand suggestSubcmd = [] for sc in module.subCommands.keys(): if(subcmd and sc[:len(subcmd)] == subcmd): suggestSubcmd.append(sc) if(len(suggestSubcmd) == 1): #only find one subcommand subcmd = suggestSubcmd[0] cmdLine = cmd + ' ' + subcmd + ' ' self.setText(cmdLine) return elif(len(suggestSubcmd) > 1): subcmd = self._suggest(subcmd, suggestSubcmd) cmdLine = cmd + ' ' + subcmd self.setText(cmdLine) utils.output(suggestSubcmd) return
def selectAll(self, sql, parameters = None): rows = [] c = self.conn.cursor() try: if(parameters): c.execute(sql, parameters) else: c.execute(sql) except: exc_type, exc_value = sys.exc_info()[:2] error = '{}: {}'.format(exc_type.__name__, exc_value) utils.output('db selectAll error: {}'.format(error), 'red') return False for r in c: rows.append(r) c.close() return rows
def selectAll(self, sql, parameters=None): rows = [] c = self.conn.cursor() try: if (parameters): c.execute(sql, parameters) else: c.execute(sql) except: exc_type, exc_value = sys.exc_info()[:2] error = '{}: {}'.format(exc_type.__name__, exc_value) utils.output('db selectAll error: {}'.format(error), 'red') return False for r in c: rows.append(r) c.close() return rows
def suggestCmd(self, cmdLine): cmds = cmdLine.split(' ') cmd = cmds[0] try: subcmd = cmds[1] except IndexError: subcmd = None param = None if (subcmd): try: param = cmds[2] except IndexError: pass if (cmd and (not subcmd)): #only have command #try to suggest command suggestArr = [] for c in utils.getCmdList(): if (c[:len(cmd)] == cmd): suggestArr.append(c) if (len(suggestArr) == 1 ): #now we got the only one correct command cmd = suggestArr[0] cmdLine = cmd + ' ' if (self.text() == cmdLine): #need suggest subcommand self.suggestSubcmd(cmd, None) else: self.setText(cmdLine) return elif (len(suggestArr) > 1): #output all possible cmds cmd = self._suggest(cmd, suggestArr) cmdLine = cmd self.setText(cmdLine) utils.output(suggestArr) return #now we have command and subcommand if (subcmd and (not param)): self.suggestSubcmd(cmd, subcmd)
def connect(self): try: self.conn = sqlite3.connect(self.dbfile) self.conn.row_factory = dict_factory self.cursor = self.conn.cursor() except: exc_type, exc_value = sys.exc_info()[:2] error = '{}: {}'.format(exc_type.__name__, exc_value) utils.output('db connect error: {}'.format(error), 'red') return False sqlList = [] sqlList.append('PRAGMA cache_size =8000') sqlList.append('PRAGMA synchronous = 0') sqlList.append('PRAGMA encoding = "UTF-8"') for sql in sqlList: self.query(sql) self.commit() return self.conn
def suggestCmd(self, cmdLine): cmds = cmdLine.split(' ') cmd = cmds[0] try: subcmd = cmds[1] except IndexError: subcmd = None param = None if(subcmd): try: param = cmds[2] except IndexError: pass if(cmd and (not subcmd)): #only have command #try to suggest command suggestArr = [] for c in utils.getCmdList(): if(c[:len(cmd)] == cmd): suggestArr.append(c) if(len(suggestArr) == 1): #now we got the only one correct command cmd = suggestArr[0] cmdLine = cmd + ' ' if(self.text() == cmdLine): #need suggest subcommand self.suggestSubcmd(cmd, None) else: self.setText(cmdLine) return elif(len(suggestArr) > 1): #output all possible cmds cmd = self._suggest(cmd, suggestArr) cmdLine = cmd self.setText(cmdLine) utils.output(suggestArr) return #now we have command and subcommand if(subcmd and (not param)): self.suggestSubcmd(cmd, subcmd)
def run(subcmd, params): if(subcmd == 'list'): setting = {} for k in subCommands.keys(): if(k != 'list'): setting[k] = globals.__dict__.get(k) utils.output(setting) elif(subcmd == 'realtime'): if(len(params) < 1): msg = 'realtime on' globals.realtime = True utils.output(msg) elif(len(params) == 1 and (params[0]=='on' or params[0]=='off')): if(params[0]=='on'): msg = 'realtime on' globals.realtime = True else: msg = 'realtime off' globals.realtime = False utils.output(msg) else: utils.output('参数错误', 'red') utils.output(subCommands['realtime']) if(not globals.realtime): chart = globals.mainwin.chart hqinfo = chart.rt_data.get('data') if(hqinfo and chart.rt_data.get('status') == 'realtime'): chart.rt_data['status'] = '' if(chart.kdata[-1]['date'] == hqinfo['date']): chart.kdata = chart.kdata[:-1] globals.mainwin.scrollbar.setMaximum(len(chart.kdata)) utils.update()
def run(subcmd, params): if (subcmd == 'shanghai_a_count'): return _getShanghaiACount() if (subcmd == 'hq'): if (len(params) != 1): utils.output('请给出股票代码', 'red') utils.output(subCommands['hq']) return stock_code = stock.normalizeStockCode(params[0]) if (not stock_code): utils.output('股票代码错误', 'red') utils.output(subCommands['hq']) return return _getHq(stock_code)
def run(subcmd, params): if(subcmd == 'shanghai_a_count'): return _getShanghaiACount() if(subcmd == 'hq'): if(len(params) != 1): utils.output('请给出股票代码', 'red') utils.output(subCommands['hq']) return stock_code = stock.normalizeStockCode(params[0]) if(not stock_code): utils.output('股票代码错误', 'red') utils.output(subCommands['hq']) return return _getHq(stock_code)
def _suggestCode(text): escapeTxt = urllib.parse.quote(text) url = 'http://suggest3.sinajs.cn/suggest/type=&key={}'.format(escapeTxt) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return 'eee' content = content.decode('gbk') codeArr = [] resultArr = re.findall(r'"(.*)"', content)[0].split(';') for line in resultArr: cArr = line.split(',') if (len(cArr) == 6 and cArr[1] == '11'): #cArr[1]==11 A股 codestr = '{} {}'.format(cArr[3], cArr[4]) codeArr.append(codestr) if (len(codeArr) < 1): utils.output('nothing') else: utils.output(codeArr) return
def downloadStockHisData(): threadName = 'download_stockhisdata' globals.data[threadName+'_threadlist'] = None globals.data[threadName+'_running'] = True threadCount = 10 start_time = datetime.datetime.now() stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' stocklist = stock.getStockList() if((not stocklist) or len(stocklist)<10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #build fetch list in_queue = queue.Queue() stockcodes = list(stocklist.keys()) stockcodes.sort() for scode in stockcodes: stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if(scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' #check diskcache before put to download queue in_queue.put(startUrl) def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName+'_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while(globals.data[threadName+'_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if(err): utils.output(err, 'red') downloadErrorCnt +=1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page +1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt downloadErrorCnt = downloadData(in_queue) if(not globals.data[threadName+'_running']): utils.output('下载已被终止', 'red') return if(downloadErrorCnt != 0): utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt)) return utils.output('下载完成')
def _updateStockList(): threadName = 'update_stocklist' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR=' page = 1 url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1)) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False #get basic info content = content.decode('gbk') totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content) if (len(totalpagelist) < 1): utils.output('取得总页数(totalpage)出错', 'red') return False totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content) if (len(totalstocklist) < 1): err = '取得总股票数(totalstock)出错' utils.output(err, 'red') return False totalpage = int(totalpagelist[0]) totalstock = int(totalstocklist[0]) utils.output('共{}页, {}只股票'.format(totalpage, totalstock)) #get real stock list data in_queue = queue.Queue() out_queue = queue.Queue() for i in range(1, totalpage + 1): url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1)) in_queue.put(url) #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>' myre = re.compile(pattern, re.DOTALL) stocklist = [] tipstr = 'update stocklist 下载' #page downloading while (page <= totalpage): try: err, u, content = out_queue.get(False) if (err): utils.output(err, 'red') stopThread(threadList) return False outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage) if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr): utils.output(outstr) else: utils.overwrite(outstr) page = page + 1 text = content.decode('gbk') #find stock list results = myre.findall(text) if (not results): err = '无法提取到股票列表数据' utils.output(err, 'red') stopThread(threadList) return False for r in results: lst = list(r) line = '{},{}'.format(lst[3], lst[6]) stocklist.append(line) except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() #download finish, stock list ready if (len(stocklist) != totalstock): err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock) utils.output(err, 'red') return False stocklist.sort() utils.output('获取股票列表完成: {}只'.format(len(stocklist))) #save data filename = globals.datapath + '/stocklist.txt' data = "\r\n".join(stocklist) if (not utils.file_put_contents(filename, data)): utils.output('数据保存失败', 'red') return False utils.output('数据保存成功') end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if((not stocklist) or len(stocklist)<10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if(scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if(err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt +=1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if(len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if(len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if(len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist=[] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if(not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, )) if(row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if(len(hisDataResults)< 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if(stype == 'stock' and len(sdata)!=7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif(stype == 'index' and len(sdata)!=6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if(stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def run(subcmd, params): utils.output(threading.enumerate())
def _updateStockList(): threadName = 'update_stocklist' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR=' page = 1 url = '{}{}'.format(partUrl, ((page-1)*50+1)) err, u, content = utils.getWebContent(url) if(err): utils.output(err, 'red') return False #get basic info content = content.decode('gbk') totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content) if(len(totalpagelist)<1): utils.output('取得总页数(totalpage)出错', 'red') return False totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content) if(len(totalstocklist)<1): err = '取得总股票数(totalstock)出错' utils.output(err, 'red') return False totalpage = int(totalpagelist[0]) totalstock = int(totalstocklist[0]) utils.output('共{}页, {}只股票'.format(totalpage, totalstock)) #get real stock list data in_queue = queue.Queue() out_queue = queue.Queue() for i in range(1, totalpage+1): url = '{}{}'.format(partUrl, ((i-1)*50+1)) in_queue.put(url) #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>' myre = re.compile(pattern, re.DOTALL) stocklist = [] tipstr = 'update stocklist 下载' #page downloading while(page <= totalpage): try: err, u, content = out_queue.get(False) if(err): utils.output(err, 'red') stopThread(threadList) return False outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage) if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr): utils.output(outstr) else: utils.overwrite(outstr) page = page +1 text = content.decode('gbk') #find stock list results = myre.findall(text) if(not results): err = '无法提取到股票列表数据' utils.output(err, 'red') stopThread(threadList) return False for r in results: lst = list(r) line = '{},{}'.format(lst[3], lst[6]) stocklist.append(line) except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() #download finish, stock list ready if(len(stocklist) != totalstock): err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock) utils.output(err, 'red') return False stocklist.sort() utils.output('获取股票列表完成: {}只'.format(len(stocklist))) #save data filename = globals.datapath + '/stocklist.txt' data = "\r\n".join(stocklist) if(not utils.file_put_contents(filename, data)): utils.output('数据保存失败', 'red') return False utils.output('数据保存成功') end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def run(subcmd, params): cmdList = utils.getCmdList() utils.output(desc) utils.output(cmdList)
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if ((not stocklist) or len(stocklist) < 10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile( "<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if (scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if (err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt += 1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if (len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if (len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if (len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist = [] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if (not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne( 'select * from stockhistory where code =? order by date desc limit 1', (scode, )) if (row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if (len(hisDataResults) < 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if (getYearAndJidu(lastupdate) == getYearAndJidu( datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if (stype == 'stock' and len(sdata) != 7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif (stype == 'index' and len(sdata) != 6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if (stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query( 'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume, qamount, qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def keyPressEvent(self, event): super().keyPressEvent(event) key = event.key() if (key == Qt.Key_Escape): globals.mainwin.chart.setFocus(Qt.OtherFocusReason) globals.mainwin.removeDockWidget(globals.mainwin.dockOne) elif (key == Qt.Key_Return): res = re.findall(r'(\S+)', self.text()) text = ' '.join(res) if (text != ''): #add command history if (not self.cmdHistory or self.cmdHistory[len(self.cmdHistory) - 1] != text): self.cmdHistory.append(text) length = len(self.cmdHistory) if (length > 100): self.cmdHistory = self.cmdHistory[length - 50:] self.cmdHistoryIdx = len(self.cmdHistory) #run command utils.output(text, 'blue') self.clear() utils.runCmd(text) elif (key == Qt.Key_U): if (event.modifiers() == Qt.ControlModifier): pos = self.cursorPosition() txt = self.text()[pos:] self.setText(txt) self.setCursorPosition(0) elif (key == Qt.Key_K): if (event.modifiers() == Qt.ControlModifier): pos = self.cursorPosition() txt = self.text()[:pos] self.setText(txt) elif (key == Qt.Key_Up): idx = self.cmdHistoryIdx - 1 if (idx < 0): idx = len(self.cmdHistory) - 1 try: self.setText(self.cmdHistory[idx]) self.cmdHistoryIdx = idx except: pass elif (key == Qt.Key_Down): idx = self.cmdHistoryIdx + 1 if (idx > len(self.cmdHistory) - 1): idx = 0 try: self.setText(self.cmdHistory[idx]) self.cmdHistoryIdx = idx except: pass elif (key == Qt.Key_Tab): res = re.findall(r'(\S+)', self.text()) text = ' '.join(res) if (text != ''): self.suggestCmd(text)
def downloadStockHisData(): threadName = 'download_stockhisdata' globals.data[threadName + '_threadlist'] = None globals.data[threadName + '_running'] = True threadCount = 10 start_time = datetime.datetime.now() stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' stocklist = stock.getStockList() if ((not stocklist) or len(stocklist) < 10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #build fetch list in_queue = queue.Queue() stockcodes = list(stocklist.keys()) stockcodes.sort() for scode in stockcodes: stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if (scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' #check diskcache before put to download queue in_queue.put(startUrl) def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName + '_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while (globals.data[threadName + '_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if (err): utils.output(err, 'red') downloadErrorCnt += 1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format( tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page + 1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt downloadErrorCnt = downloadData(in_queue) if (not globals.data[threadName + '_running']): utils.output('下载已被终止', 'red') return if (downloadErrorCnt != 0): utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt)) return utils.output('下载完成')
def run(subcmd, params): chart = globals.mainwin.chart stocklist = stock.getStockList() if (subcmd == 'di'): filterResults = [] di = indicators.di.Di() utils.output('开始di条件过滤') prefixstr = 'filter di条件过滤:' totalstock = len(stocklist) cnt = 0 start_time = datetime.datetime.now() for stockcode in stocklist: cnt += 1 stockcode = stock.normalizeStockCode(stockcode) end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format( prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr) utils.overwrite(outstr, prefixstr) stockdata = stock.getStockDayDataQianFuQuan(stockcode) stockdata = stockdata[-500:] didata = di.calculateData(stockdata) QCoreApplication.processEvents() if (didata[-1] and didata[-1].get('tip')): filterResults.append({ 'code': stockcode, 'date': didata[-1]['date'] }) dataArr = [] for row in filterResults: scode = row['code'] sdate = row['date'] row = [] row.append(scode) row.append(stock.getStockName(scode)) row.append(sdate) dataArr.append(row) #write to file begin filename = globals.datapath + '/lastFilterResult.dat' try: f = open(filename, 'wb') except IOError: exc_type, exc_value = sys.exc_info()[:2] errmsg = '{}: {}'.format(exc_type.__name__, exc_value) output(errmsg, 'red') return False pickle.dump(dataArr, f) f.close() #write to file end tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None) tbResults = globals.mainwin.stockfilterresultdlg.tbResults tbResults.setModel(tablemodel) globals.mainwin.windowStockFilterResult()
def keyPressEvent(self, event): super().keyPressEvent(event) key = event.key() if(key == Qt.Key_Escape): globals.mainwin.chart.setFocus(Qt.OtherFocusReason) globals.mainwin.removeDockWidget(globals.mainwin.dockOne) elif(key == Qt.Key_Return): res = re.findall(r'(\S+)', self.text()) text = ' '.join(res) if(text!=''): #add command history if(not self.cmdHistory or self.cmdHistory[len(self.cmdHistory)-1] != text): self.cmdHistory.append(text) length = len(self.cmdHistory) if(length>100): self.cmdHistory = self.cmdHistory[length-50:] self.cmdHistoryIdx = len(self.cmdHistory) #run command utils.output(text,'blue') self.clear() utils.runCmd(text) elif(key == Qt.Key_U): if(event.modifiers() == Qt.ControlModifier): pos = self.cursorPosition() txt = self.text()[pos:] self.setText(txt) self.setCursorPosition(0) elif(key == Qt.Key_K): if(event.modifiers() == Qt.ControlModifier): pos = self.cursorPosition() txt = self.text()[:pos] self.setText(txt) elif(key == Qt.Key_Up): idx = self.cmdHistoryIdx - 1 if(idx < 0): idx = len(self.cmdHistory) - 1 try: self.setText(self.cmdHistory[idx]) self.cmdHistoryIdx = idx except: pass elif(key == Qt.Key_Down): idx = self.cmdHistoryIdx + 1 if(idx > len(self.cmdHistory) -1): idx = 0 try: self.setText(self.cmdHistory[idx]) self.cmdHistoryIdx = idx except: pass elif(key == Qt.Key_Tab): res = re.findall(r'(\S+)', self.text()) text = ' '.join(res) if(text!=''): self.suggestCmd(text)