def getStockHq(code_list): hqData = [] url = 'http://hq.sinajs.cn/rn={}&list={}'.format(random.randint(1,1000000), code_list) err, u, content = utils.getWebContent(url) if(err): print("Error: get remote content: " + url + err) return False content = content.decode('gbk') hqList = re.findall(r'var hq_str_(.*)="(.*)";', content) for hq in hqList: code, data = hq dataArr = data.split(',') info = {} if(code[:2] == 'sh' or code[:2] == 'sz'): info = sh_sz_info(dataArr) elif(code[:2] == 'hk'): info = hk_info(dataArr) elif(code[:5] == 'rt_hk'): info = hk_info(dataArr) elif(code[:6] == 'CFF_IF'): info = if_info(dataArr) elif(code[:6] == code and re.match('^[A-Z]+$',code)): info = forex_info(dataArr) else : info = qihuo_info(dataArr) info['code'] = code if(code[:5] == 'rt_hk'): info['code'] = code[3:] hqData.append(info) return hqData
def getHq(stock_code): url = 'http://hq.sinajs.cn/rn={}&list={}'.format( random.randint(1, 1000000), stock_code) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False content = content.decode('gbk') dataArr = re.findall(r'"(.*)"', content)[0].split(',') if (len(dataArr) < 31): utils.output('no data') return False info = {} info['name'] = dataArr[0] info['code'] = stock_code info['open'] = float(dataArr[1]) info['close_yesterday'] = float(dataArr[2]) info['price'] = float(dataArr[3]) info['close'] = info['price'] info['high'] = float(dataArr[4]) info['low'] = float(dataArr[5]) info['amount'] = float(dataArr[9]) / 1000 info['date2'] = dataArr[30] info['date'] = info['date2'].replace('-', '') info['time'] = dataArr[31] if info['close_yesterday'] > 0: info['price_chg'] = (info['price'] - info['close_yesterday'] ) * 100 / info['close_yesterday'] if (info['open'] == 0): info['open'] = info['price'] return info
def _getShanghaiACount(): url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a' err, u, content = utils.getWebContent(url) if(err): utils.output(err, 'red') return False content = content.decode('gbk') cnt = re.findall(r'"([0-9]+)"', content) utils.output(cnt)
def _getShanghaiACount(): url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeStockCount?node=sh_a' err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False content = content.decode('gbk') cnt = re.findall(r'"([0-9]+)"', content) utils.output(cnt)
def _suggestCode(text): escapeTxt = urllib.parse.quote(text) url = 'http://suggest3.sinajs.cn/suggest/type=&key={}'.format(escapeTxt) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return 'eee' content = content.decode('gbk') codeArr = [] resultArr = re.findall(r'"(.*)"', content)[0].split(';') for line in resultArr: cArr = line.split(',') if (len(cArr) == 6 and cArr[1] == '11'): #cArr[1]==11 A股 codestr = '{} {}'.format(cArr[3], cArr[4]) codeArr.append(codestr) if (len(codeArr) < 1): utils.output('nothing') else: utils.output(codeArr) return
def getStockHq(code_list): hqData = [] url = 'http://hq.sinajs.cn/rn={}&list={}'.format( random.randint(1, 1000000), code_list) err, u, content = utils.getWebContent(url) if (err): print("Error: get remote content: " + url + err) return False content = content.decode('gbk') hqList = re.findall(r'var hq_str_(.*)="(.*)";', content) for hq in hqList: code, data = hq dataArr = data.split(',') info = {} if (code[:2] == 'sh' or code[:2] == 'sz'): info = sh_sz_info(dataArr) elif (code[:2] == 'hk'): info = hk_info(dataArr) elif (code[:5] == 'rt_hk'): info = hk_info(dataArr) elif (code[:6] == 'CFF_IF'): info = if_info(dataArr) elif (code[:6] == code and re.match('^[A-Z]+$', code)): info = forex_info(dataArr) else: info = qihuo_info(dataArr) info['code'] = code if (code[:5] == 'rt_hk'): info['code'] = code[3:] hqData.append(info) return hqData
def _updateStockList(): threadName = 'update_stocklist' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR=' page = 1 url = '{}{}'.format(partUrl, ((page-1)*50+1)) err, u, content = utils.getWebContent(url) if(err): utils.output(err, 'red') return False #get basic info content = content.decode('gbk') totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content) if(len(totalpagelist)<1): utils.output('取得总页数(totalpage)出错', 'red') return False totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content) if(len(totalstocklist)<1): err = '取得总股票数(totalstock)出错' utils.output(err, 'red') return False totalpage = int(totalpagelist[0]) totalstock = int(totalstocklist[0]) utils.output('共{}页, {}只股票'.format(totalpage, totalstock)) #get real stock list data in_queue = queue.Queue() out_queue = queue.Queue() for i in range(1, totalpage+1): url = '{}{}'.format(partUrl, ((i-1)*50+1)) in_queue.put(url) #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>' myre = re.compile(pattern, re.DOTALL) stocklist = [] tipstr = 'update stocklist 下载' #page downloading while(page <= totalpage): try: err, u, content = out_queue.get(False) if(err): utils.output(err, 'red') stopThread(threadList) return False outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage) if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr): utils.output(outstr) else: utils.overwrite(outstr) page = page +1 text = content.decode('gbk') #find stock list results = myre.findall(text) if(not results): err = '无法提取到股票列表数据' utils.output(err, 'red') stopThread(threadList) return False for r in results: lst = list(r) line = '{},{}'.format(lst[3], lst[6]) stocklist.append(line) except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() #download finish, stock list ready if(len(stocklist) != totalstock): err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock) utils.output(err, 'red') return False stocklist.sort() utils.output('获取股票列表完成: {}只'.format(len(stocklist))) #save data filename = globals.datapath + '/stocklist.txt' data = "\r\n".join(stocklist) if(not utils.file_put_contents(filename, data)): utils.output('数据保存失败', 'red') return False utils.output('数据保存成功') end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if((not stocklist) or len(stocklist)<10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile("<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if(scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if(err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt +=1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if(len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if(len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if(len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist=[] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if(not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne('select * from stockhistory where code =? order by date desc limit 1', (scode, )) if(row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if(len(hisDataResults)< 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if(getYearAndJidu(lastupdate) == getYearAndJidu(datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if(stype == 'stock' and len(sdata)!=7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif(stype == 'index' and len(sdata)!=6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if(stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query('replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid,scode,qdate,qopen,qhigh,qclose,qlow,qvolume,qamount,qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def _updateStockList(): threadName = 'update_stocklist' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR=' page = 1 url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1)) err, u, content = utils.getWebContent(url) if (err): utils.output(err, 'red') return False #get basic info content = content.decode('gbk') totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content) if (len(totalpagelist) < 1): utils.output('取得总页数(totalpage)出错', 'red') return False totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content) if (len(totalstocklist) < 1): err = '取得总股票数(totalstock)出错' utils.output(err, 'red') return False totalpage = int(totalpagelist[0]) totalstock = int(totalstocklist[0]) utils.output('共{}页, {}只股票'.format(totalpage, totalstock)) #get real stock list data in_queue = queue.Queue() out_queue = queue.Queue() for i in range(1, totalpage + 1): url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1)) in_queue.put(url) #start multi-thread fetching threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName) pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>' myre = re.compile(pattern, re.DOTALL) stocklist = [] tipstr = 'update stocklist 下载' #page downloading while (page <= totalpage): try: err, u, content = out_queue.get(False) if (err): utils.output(err, 'red') stopThread(threadList) return False outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage) if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr): utils.output(outstr) else: utils.overwrite(outstr) page = page + 1 text = content.decode('gbk') #find stock list results = myre.findall(text) if (not results): err = '无法提取到股票列表数据' utils.output(err, 'red') stopThread(threadList) return False for r in results: lst = list(r) line = '{},{}'.format(lst[3], lst[6]) stocklist.append(line) except queue.Empty: time.sleep(0.01) QCoreApplication.processEvents() #download finish, stock list ready if (len(stocklist) != totalstock): err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock) utils.output(err, 'red') return False stocklist.sort() utils.output('获取股票列表完成: {}只'.format(len(stocklist))) #save data filename = globals.datapath + '/stocklist.txt' data = "\r\n".join(stocklist) if (not utils.file_put_contents(filename, data)): utils.output('数据保存失败', 'red') return False utils.output('数据保存成功') end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)
def _updateStockData(): threadName = 'update_stockdata' threadCount = 10 start_time = datetime.datetime.now() #terminal all thread from threadList def stopThread(threadList): for t in threadList: t.stop() stocklist = stock.getStockList() if ((not stocklist) or len(stocklist) < 10): utils.output('获取股票列表失败', 'red') return False utils.output('获取股票列表成功, 数目: {}'.format(len(stocklist))) stockStartUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{}.phtml' indexStartUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{}/type/S.phtml' baseStockUrl = 'http://money.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/{code}.phtml?year={year}&jidu={jidu}' baseIndexUrl = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{code}/type/S.phtml?year={year}&jidu={jidu}' #add index 上证50 and 上证综指 stocklist['sh000001'] = '上证综指' stocklist['sh000016'] = '上证50' #re for match stock history data reStockHisMatch = re.compile( "<a target='_blank' href='http://vip.stock.finance.sina.com.cn/quotes_service/view/vMS_tradehistory.php\?symbol=(.+?)&date=(.+?)'>(.+?)</tr>", re.S) skeys = list(stocklist.keys()) skeys.sort() scnt = 1 stotal = len(skeys) for scode in skeys: sname = stocklist[scode] stype = 'stock' startUrl = stockStartUrl.format(scode[2:]) if (scode[:3] == 'sh0'): #大盘指数 startUrl = indexStartUrl.format(scode[2:]) stype = 'index' err, u, content = utils.getWebContent(startUrl) if (err): utils.output(err, 'red') return False content = content.decode('gbk') utils.output('开始更新数据: {}/{} {} {}'.format(scnt, stotal, scode, sname)) scnt += 1 stocksection = re.findall(r'<!--历史交易begin-->(.+?)<!--历史交易end-->', content, re.S) if (len(stocksection) != 1): utils.output('截取股票历史数据失败 {}'.format(u), 'red') return False stockyearsecton = re.findall(r'<select name="year">(.+?)</select>', stocksection[0], re.S) if (len(stockyearsecton) != 1): utils.output('截取股票历史数据:年份数据失败 {}'.format(u), 'red') return False yearsdata = re.findall(r'<option value="(\d+?)"(.+?)</option>', stockyearsecton[0], re.S) if (len(yearsdata) < 1): utils.output('截取股票历史数据:提取年份失败 {}'.format(u), 'red') return False yearlist = [] for y in yearsdata: yearlist.append(list(y)[0]) #check last update for this stock lastupdate = None lite3db = db.Sqlite3() if (not lite3db.connect()): utils.output('截取股票历史数据:数据库连接失败', 'red') return False row = lite3db.selectOne( 'select * from stockhistory where code =? order by date desc limit 1', (scode, )) if (row): lastupdate = row['date'] else: lastupdate = '2000-01-01' lastupdate = '2011-01-01' #for TEST~~ hisDataResults = reStockHisMatch.findall(stocksection[0]) if (len(hisDataResults) < 1): utils.output('截取股票历史数据:匹配具体数据失败', 'red') return False #webLastStockDate = hisDataResults[0][1] #if(lastupdate == webLastStockDate): # utils.output('数据库内数据已是最新: {}'.format(lastupdate)) # continue #start download history data if (getYearAndJidu(lastupdate) == getYearAndJidu( datetime.date.today().isoformat())): for res in hisDataResults: sdata = re.findall(r'<div align="center">(.+?)</div></td>', res[2]) if (stype == 'stock' and len(sdata) != 7): utils.output('截取股票历史数据:数据格式错误1', 'red') return False elif (stype == 'index' and len(sdata) != 6): utils.output('截取股票历史数据:数据格式错误2', 'red') return False qdate = res[1] qopen = sdata[0] qhigh = sdata[1] qclose = sdata[2] qlow = sdata[3] qvolume = sdata[4] qamount = sdata[5] qweight = '1.0' if (stype == 'stock'): qweight = sdata[6] qid = scode + qdate lite3db.query( 'replace into stockhistory(id,code,date,open,high,close,low,volume,amount,weight)values \ (?,?,?,?,?,?,?,?,?,?)', (qid, scode, qdate, qopen, qhigh, qclose, qlow, qvolume, qamount, qweight)) lite3db.commit() end_time = datetime.datetime.now() return '{}'.format(end_time - start_time)