def run(subcmd, params): chart = globals.mainwin.chart stocklist = stock.getStockList() if(subcmd == 'di'): filterResults = [] di = indicators.di.Di() utils.output('开始di条件过滤') prefixstr = 'filter di条件过滤:' totalstock = len(stocklist) cnt = 0 start_time = datetime.datetime.now() for stockcode in stocklist: cnt +=1 stockcode = stock.normalizeStockCode(stockcode) end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr) utils.overwrite(outstr, prefixstr) stockdata = stock.getStockDayDataQianFuQuan(stockcode) stockdata = stockdata[-500:] didata = di.calculateData(stockdata) QCoreApplication.processEvents() if(didata[-1] and didata[-1].get('tip')): filterResults.append({'code':stockcode, 'date':didata[-1]['date']}) dataArr = [] for row in filterResults: scode = row['code'] sdate = row['date'] row=[] row.append(scode) row.append(stock.getStockName(scode)) row.append(sdate) dataArr.append(row) #write to file begin filename = globals.datapath + '/lastFilterResult.dat' try: f = open(filename, 'wb') except IOError: exc_type, exc_value = sys.exc_info()[:2] errmsg = '{}: {}'.format(exc_type.__name__, exc_value) output(errmsg, 'red') return False pickle.dump(dataArr, f) f.close() #write to file end tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None) tbResults = globals.mainwin.stockfilterresultdlg.tbResults tbResults.setModel(tablemodel) globals.mainwin.windowStockFilterResult()
def run(subcmd, params): chart = globals.mainwin.chart stocklist = stock.getStockList() if (subcmd == 'di'): filterResults = [] di = indicators.di.Di() utils.output('开始di条件过滤') prefixstr = 'filter di条件过滤:' totalstock = len(stocklist) cnt = 0 start_time = datetime.datetime.now() for stockcode in stocklist: cnt += 1 stockcode = stock.normalizeStockCode(stockcode) end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format( prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr) utils.overwrite(outstr, prefixstr) stockdata = stock.getStockDayDataQianFuQuan(stockcode) stockdata = stockdata[-500:] didata = di.calculateData(stockdata) QCoreApplication.processEvents() if (didata[-1] and didata[-1].get('tip')): filterResults.append({ 'code': stockcode, 'date': didata[-1]['date'] }) dataArr = [] for row in filterResults: scode = row['code'] sdate = row['date'] row = [] row.append(scode) row.append(stock.getStockName(scode)) row.append(sdate) dataArr.append(row) #write to file begin filename = globals.datapath + '/lastFilterResult.dat' try: f = open(filename, 'wb') except IOError: exc_type, exc_value = sys.exc_info()[:2] errmsg = '{}: {}'.format(exc_type.__name__, exc_value) output(errmsg, 'red') return False pickle.dump(dataArr, f) f.close() #write to file end tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None) tbResults = globals.mainwin.stockfilterresultdlg.tbResults tbResults.setModel(tablemodel) globals.mainwin.windowStockFilterResult()
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if((not stocklist) or len(stocklist)<10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if(scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if(err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt +=1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if(len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if(len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if(len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist=[] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if(not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, )) if(row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if(len(hisDataResults)< 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if(stype == 'stock' and len(sdata)!=7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif(stype == 'index' and len(sdata)!=6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if(stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if ((not stocklist) or len(stocklist) < 10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile( "<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if (scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if (err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt += 1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if (len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if (len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if (len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist = [] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if (not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne( 'select * from stockhistory where code =? order by date desc limit 1', (scode, )) if (row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if (len(hisDataResults) < 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if (getYearAndJidu(lastupdate) == getYearAndJidu( datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if (stype == 'stock' and len(sdata) != 7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif (stype == 'index' and len(sdata) != 6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if (stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query( 'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume, qamount, qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def downloadStockHisData(): threadName = 'download_stockhisdata' globals.data[threadName + '_threadlist'] = None globals.data[threadName + '_running'] = True threadCount = 10 start_time = datetime.datetime.now() stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' stocklist = stock.getStockList() if ((not stocklist) or len(stocklist) < 10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #build fetch list in_queue = queue.Queue() stockcodes = list(stocklist.keys()) stockcodes.sort() for scode in stockcodes: stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if (scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' #check diskcache before put to download queue in_queue.put(startUrl) def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName + '_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while (globals.data[threadName + '_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if (err): utils.output(err, 'red') downloadErrorCnt += 1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format( tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page + 1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt downloadErrorCnt = downloadData(in_queue) if (not globals.data[threadName + '_running']): utils.output('下载已被终止', 'red') return if (downloadErrorCnt != 0): utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt)) return utils.output('下载完成')
def downloadStockHisData(): threadName = 'download_stockhisdata' globals.data[threadName+'_threadlist'] = None globals.data[threadName+'_running'] = True threadCount = 10 start_time = datetime.datetime.now() stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' stocklist = stock.getStockList() if((not stocklist) or len(stocklist)<10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #build fetch list in_queue = queue.Queue() stockcodes = list(stocklist.keys()) stockcodes.sort() for scode in stockcodes: stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if(scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' #check diskcache before put to download queue in_queue.put(startUrl) def downloadData(in_queue): out_queue = queue.Queue() page = 1 totalpage = in_queue.qsize() #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) globals.data[threadName+'_threadlist'] = threadList #page downloading downloadErrorCnt = 0 while(globals.data[threadName+'_running'] and (page <= totalpage)): try: err, u, content = out_queue.get(False) if(err): utils.output(err, 'red') downloadErrorCnt +=1 continue end_time = datetime.datetime.now() diffStr = '{}'.format(end_time - start_time) tipstr = '{} 下载'.format(threadName) outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr) utils.overwrite(outstr, tipstr) page = page +1 except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() return downloadErrorCnt downloadErrorCnt = downloadData(in_queue) if(not globals.data[threadName+'_running']): utils.output('下载已被终止', 'red') return if(downloadErrorCnt != 0): utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt)) return utils.output('下载完成')