def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
globals.data[threadName + '_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while (globals.data[threadName + '_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
downloadErrorCnt += 1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(
tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page + 1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def downloadData(in_queue):
out_queue = queue.Queue()
page = 1
totalpage = in_queue.qsize()
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
globals.data[threadName+'_threadlist'] = threadList
#page downloading
downloadErrorCnt = 0
while(globals.data[threadName+'_running'] and (page <= totalpage)):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
downloadErrorCnt +=1
continue
end_time = datetime.datetime.now()
diffStr = '{}'.format(end_time - start_time)
tipstr = '{} 下载'.format(threadName)
outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
utils.overwrite(outstr, tipstr)
page = page +1
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
return downloadErrorCnt
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page-1)*50+1))
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if(len(totalpagelist)<1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if(len(totalstocklist)<1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage+1):
url = '{}{}'.format(partUrl, ((i-1)*50+1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while(page <= totalpage):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage)
if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page +1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if(not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if(len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if(not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1))
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if (len(totalpagelist) < 1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if (len(totalstocklist) < 1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage + 1):
url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while (page <= totalpage):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page,
totalpage)
if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page + 1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if (not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if (len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist),
totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if (not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
