def getStockHq(code_list):
hqData = []
url = 'http://hq.sinajs.cn/rn={}&list={}'.format(random.randint(1,1000000), code_list)
err, u, content = utils.getWebContent(url)
if(err):
print("Error: get remote content: " + url + err)
return False
content = content.decode('gbk')
hqList = re.findall(r'var hq_str_(.*)="(.*)";', content)
for hq in hqList:
code, data = hq
dataArr = data.split(',')
info = {}
if(code[:2] == 'sh' or code[:2] == 'sz'):
info = sh_sz_info(dataArr)
elif(code[:2] == 'hk'):
info = hk_info(dataArr)
elif(code[:5] == 'rt_hk'):
info = hk_info(dataArr)
elif(code[:6] == 'CFF_IF'):
info = if_info(dataArr)
elif(code[:6] == code and re.match('^[A-Z]+$',code)):
info = forex_info(dataArr)
else :
info = qihuo_info(dataArr)
info['code'] = code
if(code[:5] == 'rt_hk'):
info['code'] = code[3:]
hqData.append(info)
return hqData
def getHq(stock_code):
url = 'http://hq.sinajs.cn/rn={}&list={}'.format(
random.randint(1, 1000000), stock_code)
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
dataArr = re.findall(r'"(.*)"', content)[0].split(',')
if (len(dataArr) < 31):
utils.output('no data')
return False
info = {}
info['name'] = dataArr[0]
info['code'] = stock_code
info['open'] = float(dataArr[1])
info['close_yesterday'] = float(dataArr[2])
info['price'] = float(dataArr[3])
info['close'] = info['price']
info['high'] = float(dataArr[4])
info['low'] = float(dataArr[5])
info['amount'] = float(dataArr[9]) / 1000
info['date2'] = dataArr[30]
info['date'] = info['date2'].replace('-', '')
info['time'] = dataArr[31]
if info['close_yesterday'] > 0:
info['price_chg'] = (info['price'] - info['close_yesterday']
) * 100 / info['close_yesterday']
if (info['open'] == 0):
info['open'] = info['price']
return info
def _getShanghaiACount():
url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a'
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
cnt = re.findall(r'"([0-9]+)"', content)
utils.output(cnt)
def _getShanghaiACount():
url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a'
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
cnt = re.findall(r'"([0-9]+)"', content)
utils.output(cnt)
def _suggestCode(text):
escapeTxt = urllib.parse.quote(text)
url = 'http://suggest3.sinajs.cn/suggest/type=&key={}'.format(escapeTxt)
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return 'eee'
content = content.decode('gbk')
codeArr = []
resultArr = re.findall(r'"(.*)"', content)[0].split(';')
for line in resultArr:
cArr = line.split(',')
if (len(cArr) == 6 and cArr[1] == '11'): #cArr[1]==11 A股
codestr = '{} {}'.format(cArr[3], cArr[4])
codeArr.append(codestr)
if (len(codeArr) < 1):
utils.output('nothing')
else:
utils.output(codeArr)
return
def getStockHq(code_list):
hqData = []
url = 'http://hq.sinajs.cn/rn={}&list={}'.format(
random.randint(1, 1000000), code_list)
err, u, content = utils.getWebContent(url)
if (err):
print("Error: get remote content: " + url + err)
return False
content = content.decode('gbk')
hqList = re.findall(r'var hq_str_(.*)="(.*)";', content)
for hq in hqList:
code, data = hq
dataArr = data.split(',')
info = {}
if (code[:2] == 'sh' or code[:2] == 'sz'):
info = sh_sz_info(dataArr)
elif (code[:2] == 'hk'):
info = hk_info(dataArr)
elif (code[:5] == 'rt_hk'):
info = hk_info(dataArr)
elif (code[:6] == 'CFF_IF'):
info = if_info(dataArr)
elif (code[:6] == code and re.match('^[A-Z]+$', code)):
info = forex_info(dataArr)
else:
info = qihuo_info(dataArr)
info['code'] = code
if (code[:5] == 'rt_hk'):
info['code'] = code[3:]
hqData.append(info)
return hqData
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page-1)*50+1))
err, u, content = utils.getWebContent(url)
if(err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if(len(totalpagelist)<1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if(len(totalstocklist)<1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage+1):
url = '{}{}'.format(partUrl, ((i-1)*50+1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while(page <= totalpage):
try:
err, u, content = out_queue.get(False)
if(err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage)
if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page +1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if(not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if(len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if(not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if((not stocklist) or len(stocklist)<10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if(scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if(err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt +=1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S)
if(len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S)
if(len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S)
if(len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist=[]
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if(not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, ))
if(row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if(len(hisDataResults)< 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2])
if(stype == 'stock' and len(sdata)!=7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif(stype == 'index' and len(sdata)!=6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if(stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockList():
threadName = 'update_stocklist'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
page = 1
url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1))
err, u, content = utils.getWebContent(url)
if (err):
utils.output(err, 'red')
return False
#get basic info
content = content.decode('gbk')
totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
if (len(totalpagelist) < 1):
utils.output('取得总页数(totalpage)出错', 'red')
return False
totalstocklist = re.findall(r'第1条到第50条，共([0-9]+)条', content)
if (len(totalstocklist) < 1):
err = '取得总股票数(totalstock)出错'
utils.output(err, 'red')
return False
totalpage = int(totalpagelist[0])
totalstock = int(totalstocklist[0])
utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
#get real stock list data
in_queue = queue.Queue()
out_queue = queue.Queue()
for i in range(1, totalpage + 1):
url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1))
in_queue.put(url)
#start multi-thread fetching
threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
threadName)
pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
myre = re.compile(pattern, re.DOTALL)
stocklist = []
tipstr = 'update stocklist 下载'
#page downloading
while (page <= totalpage):
try:
err, u, content = out_queue.get(False)
if (err):
utils.output(err, 'red')
stopThread(threadList)
return False
outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page,
totalpage)
if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
utils.output(outstr)
else:
utils.overwrite(outstr)
page = page + 1
text = content.decode('gbk')
#find stock list
results = myre.findall(text)
if (not results):
err = '无法提取到股票列表数据'
utils.output(err, 'red')
stopThread(threadList)
return False
for r in results:
lst = list(r)
line = '{},{}'.format(lst[3], lst[6])
stocklist.append(line)
except queue.Empty:
time.sleep(0.01)
QCoreApplication.processEvents()
#download finish, stock list ready
if (len(stocklist) != totalstock):
err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist),
totalstock)
utils.output(err, 'red')
return False
stocklist.sort()
utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
#save data
filename = globals.datapath + '/stocklist.txt'
data = "\r\n".join(stocklist)
if (not utils.file_put_contents(filename, data)):
utils.output('数据保存失败', 'red')
return False
utils.output('数据保存成功')
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
def _updateStockData():
threadName = 'update_stockdata'
threadCount = 10
start_time = datetime.datetime.now()
#terminal all thread from threadList
def stopThread(threadList):
for t in threadList:
t.stop()
stocklist = stock.getStockList()
if ((not stocklist) or len(stocklist) < 10):
utils.output('获取股票列表失败', 'red')
return False
utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist)))
stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml'
indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml'
baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}'
baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}'
#add index 上证50 and 上证综指
stocklist['sh000001'] = '上证综指'
stocklist['sh000016'] = '上证50'
#re for match stock history data
reStockHisMatch = re.compile(
"<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>",
re.S)
skeys = list(stocklist.keys())
skeys.sort()
scnt = 1
stotal = len(skeys)
for scode in skeys:
sname = stocklist[scode]
stype = 'stock'
startUrl = stockStartUrl.format(scode[2:])
if (scode[:3] == 'sh0'): #大盘指数
startUrl = indexStartUrl.format(scode[2:])
stype = 'index'
err, u, content = utils.getWebContent(startUrl)
if (err):
utils.output(err, 'red')
return False
content = content.decode('gbk')
utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname))
scnt += 1
stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->',
content, re.S)
if (len(stocksection) != 1):
utils.output('截取股票历史数据失败 {}'.format(u), 'red')
return False
stockyearsecton = re.findall(r'<select name="year">(.+?)</select>',
stocksection[0], re.S)
if (len(stockyearsecton) != 1):
utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red')
return False
yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>',
stockyearsecton[0], re.S)
if (len(yearsdata) < 1):
utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red')
return False
yearlist = []
for y in yearsdata:
yearlist.append(list(y)[0])
#check last update for this stock
lastupdate = None
lite3db = db.Sqlite3()
if (not lite3db.connect()):
utils.output('截取股票历史数据:数据库连接失败', 'red')
return False
row = lite3db.selectOne(
'select * from stockhistory where code =? order by date desc limit 1',
(scode, ))
if (row):
lastupdate = row['date']
else:
lastupdate = '2000-01-01'
lastupdate = '2011-01-01' #for TEST~~
hisDataResults = reStockHisMatch.findall(stocksection[0])
if (len(hisDataResults) < 1):
utils.output('截取股票历史数据:匹配具体数据失败', 'red')
return False
#webLastStockDate = hisDataResults[0][1]
#if(lastupdate == webLastStockDate):
# utils.output('数据库内数据已是最新: {}'.format(lastupdate))
# continue
#start download history data
if (getYearAndJidu(lastupdate) == getYearAndJidu(
datetime.date.today().isoformat())):
for res in hisDataResults:
sdata = re.findall(r'<div align="center">(.+?)</div></td>',
res[2])
if (stype == 'stock' and len(sdata) != 7):
utils.output('截取股票历史数据:数据格式错误1', 'red')
return False
elif (stype == 'index' and len(sdata) != 6):
utils.output('截取股票历史数据:数据格式错误2', 'red')
return False
qdate = res[1]
qopen = sdata[0]
qhigh = sdata[1]
qclose = sdata[2]
qlow = sdata[3]
qvolume = sdata[4]
qamount = sdata[5]
qweight = '1.0'
if (stype == 'stock'):
qweight = sdata[6]
qid = scode + qdate
lite3db.query(
'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \
(?,?,?,?,?,?,?,?,?,?)',
(qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume,
qamount, qweight))
lite3db.commit()
end_time = datetime.datetime.now()
return '{}'.format(end_time - start_time)
