def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if(subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt +=1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if(didata[-1] and didata[-1].get('tip')):
filterResults.append({'code':stockcode, 'date':didata[-1]['date']})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row=[]
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if (subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt += 1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(
prefixstr, cnt, totalstock, stockcode, len(filterResults),
diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if (didata[-1] and didata[-1].get('tip')):
filterResults.append({
'code': stockcode,
'date': didata[-1]['date']
})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row = []
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if((not stocklist) or len(stocklist)<10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if(scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if(err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt +=1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S)
if(len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S)
if(len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S)
if(len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist=[]
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if(not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, ))
if(row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if(len(hisDataResults)< 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2])
if(stype == 'stock' and len(sdata)!=7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif(stype == 'index' and len(sdata)!=6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if(stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if ((not stocklist) or len(stocklist) < 10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile(
"<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>",
re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if (scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt += 1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->',
content, re.S)
if (len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>',
stocksection[0], re.S)
if (len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>',
stockyearsecton[0], re.S)
if (len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist = []
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if (not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne(
'select * from stockhistory where code =? order by date desc limit 1',
(scode, ))
if (row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if (len(hisDataResults) < 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if (getYearAndJidu(lastupdate) == getYearAndJidu(
datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>',
res[2])
if (stype == 'stock' and len(sdata) != 7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif (stype == 'index' and len(sdata) != 6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if (stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query(
'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume,
qamount, qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def downloadStockHisData():
threadName = 'download_stockhisdata'
globals.data[threadName + '_threadlist'] = None
globals.data[threadName + '_running'] = True
threadCount = 10
start_time = datetime.datetime.now()
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
stocklist = stock.getStockList()
if ((not stocklist) or len(stocklist) < 10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#build fetch list
in_queue = queue.Queue()
stockcodes = list(stocklist.keys())
stockcodes.sort()
for scode in stockcodes:
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if (scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
#check diskcache before put to download queue
in_queue.put(startUrl)
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
globals.data[threadName + '_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while (globals.data[threadName + '_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
downloadErrorCnt += 1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(
tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page + 1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
downloadErrorCnt = downloadData(in_queue)
if (not globals.data[threadName + '_running']):
utils.output('下载已被终止', 'red')
return
if (downloadErrorCnt != 0):
utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt))
return
utils.output('下载完成')
def downloadStockHisData():
threadName = 'download_stockhisdata'
globals.data[threadName+'_threadlist'] = None
globals.data[threadName+'_running'] = True
threadCount = 10
start_time = datetime.datetime.now()
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
stocklist = stock.getStockList()
if((not stocklist) or len(stocklist)<10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#build fetch list
in_queue = queue.Queue()
stockcodes = list(stocklist.keys())
stockcodes.sort()
for scode in stockcodes:
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if(scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
#check diskcache before put to download queue
in_queue.put(startUrl)
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
globals.data[threadName+'_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while(globals.data[threadName+'_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
downloadErrorCnt +=1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page +1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
downloadErrorCnt = downloadData(in_queue)
if(not globals.data[threadName+'_running']):
utils.output('下载已被终止', 'red')
return
if(downloadErrorCnt != 0):
utils.output('下载出错的页面数: {}, 请重新运行命令下载出错的页面'.format(downloadErrorCnt))
return
utils.output('下载完成')
