def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
globals.data[threadName + '_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while (globals.data[threadName + '_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
downloadErrorCnt += 1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(
tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page + 1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
globals.data[threadName+'_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while(globals.data[threadName+'_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
downloadErrorCnt +=1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page +1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def cancelDownload(threadName):
def stopThread(threadName):
threadList = []
try:
threadList = globals.data[threadName+'_threadlist']
globals.data[threadName+'_threadlist'] = None
globals.data[threadName+'_running'] = False
except:
pass
if(threadList):
for t in threadList:
t.stop()
threadLeft = 0
stopThread(threadName)
for t in threading.enumerate():
if(t.getName() == threadName):
threadLeft +=1
if(threadLeft > 0):
tipstr = '终止下载中, 目前剩余'
outstr = '{} {} 线程'.format(tipstr, threadLeft)
utils.overwrite(outstr, tipstr)
return False
else:
utils.output('所有下载线程已终止')
return True
def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if(subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt +=1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if(didata[-1] and didata[-1].get('tip')):
filterResults.append({'code':stockcode, 'date':didata[-1]['date']})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row=[]
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def run(subcmd, params):
chart = globals.mainwin.chart
stocklist = stock.getStockList()
if (subcmd == 'di'):
filterResults = []
di = indicators.di.Di()
utils.output('开始di条件过滤')
prefixstr = 'filter di条件过滤:'
totalstock = len(stocklist)
cnt = 0
start_time = datetime.datetime.now()
for stockcode in stocklist:
cnt += 1
stockcode = stock.normalizeStockCode(stockcode)
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(
prefixstr, cnt, totalstock, stockcode, len(filterResults),
diffStr)
utils.overwrite(outstr, prefixstr)
stockdata = stock.getStockDayDataQianFuQuan(stockcode)
stockdata = stockdata[-500:]
didata = di.calculateData(stockdata)
QCoreApplication.processEvents()
if (didata[-1] and didata[-1].get('tip')):
filterResults.append({
'code': stockcode,
'date': didata[-1]['date']
})
dataArr = []
for row in filterResults:
scode = row['code']
sdate = row['date']
row = []
row.append(scode)
row.append(stock.getStockName(scode))
row.append(sdate)
dataArr.append(row)
#write to file begin
filename = globals.datapath + '/lastFilterResult.dat'
try:
f = open(filename, 'wb')
except IOError:
exc_type, exc_value = sys.exc_info()[:2]
errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
output(errmsg, 'red')
return False
pickle.dump(dataArr, f)
f.close()
#write to file end
tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
tbResults = globals.mainwin.stockfilterresultdlg.tbResults
tbResults.setModel(tablemodel)
globals.mainwin.windowStockFilterResult()
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page-1)*50+1))
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if(len(totalpagelist)<1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if(len(totalstocklist)<1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage+1):
url = '{}{}'.format(partUrl, ((i-1)*50+1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while(page <= totalpage):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage)
if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page +1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if(not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if(len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if(not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1))
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if (len(totalpagelist) < 1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if (len(totalstocklist) < 1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage + 1):
url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while (page <= totalpage):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page,
totalpage)
if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page + 1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if (not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if (len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist),
totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if (not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)