def QA_SU_crawl_eastmoney(action="zjlx", stockCode=None): ''' :param action: zjlx 后期支持其他的操作类型 :param stockCode: 股票代码 :return: ''' stockItems = QA_fetch_stock_list() if stockCode == "all": # 读取tushare股票列表代码 print("💪 一共需要获取 %d 个股票的 资金流向 , 需要大概 %d 小时" % (len(stockItems), (len(stockItems) * 5) / 60 / 60)) code_list = [] for stock in stockItems: code_list.append(stock['code']) # print(stock['code']) crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite( code_list) # print(stock) return else: # todo 检查股票代码是否合法 # return crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(stockCode=stockCode) code_list = [] code_list.append(stockCode) return crawl_eastmoney_file.QA_request_eastmoney_zjlx( param_stock_code_list=code_list)
def test_QThread_Check_ZJLX_DB_Count(self): qthread = QThread_Check_ZJLX_DB_Status() stockListAll = QA_fetch_stock_list() for iRec in stockListAll: stockCode = iRec['code'] lstRet = qthread.QA_count_eastmoney_stock_xjlc_record_count_one_by_one(str_stock_code=stockCode) print(lstRet)
def QA_fetch_stock_list_adv(collections=DATABASE.stock_list): ''' '获取股票列表' :param collections: mongodb 数据库 :return: DataFrame ''' stock_list_items = QA_fetch_stock_list(collections) if len(stock_list_items) == 0: print("QA Error QA_fetch_stock_list_adv call item for item in collections.find() return 0 item, maybe the DATABASE.stock_list is empty!") return None return stock_list_items
def QA_fetch_stock_list_adv(collections=DATABASE.stock_list): ''' '获取股票列表' :param collections: mongodb 数据库 :return: DataFrame ''' stock_list_items = QA_fetch_stock_list(collections) if len(stock_list_items) == 0: print( "QA Error QA_fetch_stock_list_adv call item for item in collections.find() return 0 item, maybe the DATABASE.stock_list is empty!" ) return None return pd.DataFrame(stock_list_items).drop( '_id', axis=1, inplace=False).set_index('code', drop=False)
def run(self): #time.sleep(2) print("QThread_GetStockList is running") # 检查数据库是否已经开启 try: self.stockListAll = QA_fetch_stock_list() except Exception as ee: # print(ee) self.strTaskRunningResult = ee.__str__() self.stockListAll = None self.stockCountAll = 0 return self.stockCountAll = len(self.stockListAll) self.strTaskRunningResult = "成功获取 股票列表, 🔗 数据库成功 , 共 {} 个股票 😄".format( self.stockCountAll) pass
def get(self): page = int(self.get_argument('page', default=1)) page_size = int(self.get_argument('page_size', default=10)) start = (page - 1) * page_size end = page * page_size - 1 stock_list = QA_fetch_stock_list() stock_list['pre_close'] = stock_list['pre_close'].apply( lambda x: round(x, 2)) stock_fav = QA_fetch_stock_fav() stock_list = stock_list[stock_list["code"].isin( stock_fav['code'].tolist())] total = len(stock_list) data = QA_util_to_json_from_pandas(stock_list[start:end]) return self.write({ 'result': data, 'page': int(page), 'page_size': int(page_size), 'total': int(total) })
def QA_SU_crawl_eastmoney(action="zjlx",stockCode=None, fromStockCodeToStart=None): ''' :param action: zjlx 后期支持其他的操作类型 :param stockCode: 股票代码 :return: ''' stockItems = QA_fetch_stock_list() if stockCode=="all" or stockCode== "continue": code_list = [] bCheckpointAfter = False for stock in stockItems: if stock['code'] == fromStockCodeToStart: bCheckpointAfter = True if fromStockCodeToStart != None and bCheckpointAfter == False and stock['code'] != fromStockCodeToStart: continue elif fromStockCodeToStart != None and stock['code'] == fromStockCodeToStart: bCheckpointAfter = True code_list.append(stock['code']) #print(stock['code']) #读取tushare股票列表代码 print("💪 一共需要获取 %d 个股票的 资金流向 , 需要大概 %d 小时" % (len(code_list), (len(code_list)*30)/60/60 )) crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(code_list) #print(stock) return else: #todo 检查股票代码是否合法 #return crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(stockCode=stockCode) code_list = [] code_list.append(stockCode) return crawl_eastmoney_file.QA_request_eastmoney_zjlx(param_stock_code_list=code_list)
def QA_statistic_block_data(self): # 🛠 todo 直接使用monogodb 数据库去统计 try: block_list_df = QA_fetch_stock_block() blockNameList = {} block_list_df = block_list_df.set_index(['blockname']) for strIndexBlockName in block_list_df.index: try: dict = blockNameList[strIndexBlockName] dict['count'] = dict['count'] + 1 self.strTaskRunningLog = "正在统计板块{} 股票个数{}".format(strIndexBlockName, dict['count']) except Exception as ee: dictNew = {} dictNew['up'] = 0 #上涨家数 dictNew['even'] = 0 #平盘家数 dictNew['down'] = 0 #下跌家数 dictNew['count'] = 0 #板块股票数 dictNew['upRatio'] = 0.0 #上涨家数比率 dictNew['downRation'] = 0.0 #下跌家数比率 blockNameList[strIndexBlockName] = dictNew #row2 = block_list_df[i] ####################################################################################################### strToday = QADate.QA_util_today_str() strEndDate = QADate_trade.QA_util_get_real_date(date = strToday) strStartDate = QADate_trade.QA_util_get_last_day(date= strEndDate) codeTwoDayPriceeDict = {} stock_list = QA_fetch_stock_list() stock_list_len = len(stock_list) stock_name_count = 0 for aStockCode in stock_list: stock_name_count = stock_name_count + 1 strAcode = aStockCode['code'] strAName = aStockCode['name'] priceTowDay = QA_fetch_stock_day_adv(code=strAcode, start=strStartDate, end=strEndDate) up = None if priceTowDay is not None: priceTowDay.to_qfq() # price_len = len(priceTowDay) # print(priceTowDay) price_len = len(priceTowDay) if price_len == 2: closeSerial = priceTowDay.close # todayPrice = priceTowDay[1].close v1 = closeSerial[0] v2 = closeSerial[1] if v1 < v2: up = True else: up = False codeTwoDayPriceeDict[strAcode] = {'pricetwoday':priceTowDay, 'up':up} if codeTwoDayPriceeDict[strAcode]['up'] is not None and codeTwoDayPriceeDict[strAcode]['up'] == True: self.strTaskRunningLog = "正在获取股票价格 {} {} 上涨 ⬆🔺, 进度{}/{} " \ .format(strAcode, strAName, stock_name_count, stock_list_len) elif codeTwoDayPriceeDict[strAcode]['up'] is not None and codeTwoDayPriceeDict[strAcode]['up'] == False: self.strTaskRunningLog = "正在获取股票价格 {} {} 下跌 ⬇️, 进度{}/{} " \ .format(strAcode, strAName, stock_name_count, stock_list_len) else: self.strTaskRunningLog = "正在获取股票价格 {} {} 和前一天的无法统计️, 进度{}/{} " \ .format(strAcode, strAName, stock_name_count, stock_list_len) ########################################################################################################## len_block_size = len(blockNameList) count = 0 for blockName in blockNameList.keys(): count = count + 1 stocks_in_block = QA_fetch_stock_block_adv(code=None, blockname= blockName) self.strTaskRunningLog = "正在统计板块{} ,进度{}/{}".format(blockName,count,len_block_size) code_list = stocks_in_block.code iCodeLength = len(code_list) iCountForCode = 0 for iCode in code_list: iCountForCode = iCountForCode + 1 try: priceTowDay = codeTwoDayPriceeDict[iCode]['pricetwoday'] upValue = codeTwoDayPriceeDict[iCode]['up'] if priceTowDay is not None: if upValue == True: blockNameList[blockName]['up'] = blockNameList[blockName]['up'] + 1 elif upValue == False: blockNameList[blockName]['down'] = blockNameList[blockName]['down'] + 1 except Exception as ee: strErrorMsg = ee.__str__() print(strErrorMsg) self.strTaskRunningLog = "正在统计板块{} ,进度{}/{} \n 股票 {}, 进度 {}/{}"\ .format(blockName, count, len_block_size, iCode, iCountForCode, iCodeLength) pass #print(stocks_in_block) ####################################################################################################### except Exception as eee: strErrorMsg = eee.__str__() print(strErrorMsg) finally: return blockNameList
def run(self): # def find(f, seq): """Return first item in sequence where f(item) == True.""" for item in seq: if f(item): return item self.zjlxRecNeedUpdateStockCodes.clear() self.stockListAll.clear() self.stockListAll = QA_fetch_stock_list() self.strTaskRunningLog = '正在检查数据库,已经获取的数据' itemsInZJLX = self.QA_count_eastmoney_stock_xjlc_record_count_by_aggregate( ) ''' 追究没有获取过的记录, ''' # 非常慢 # codeNeedAppend = [] # for aItemInAllStock in self.stockListAll: # code0 = aItemInAllStock['code'] # #self.strTaskRunningLog = '正在检查数据 {}'.format(code0) # # foundItem = find(lambda aRec: aRec['_id'] == code0, itemsInZJLX) # if foundItem is None: # codeNeedAppend.append(code0) # # # print(codeNeedAppend) # 使用 dict allItemsByDict = {} partItemsByDict = {} needToAppendCode = [] self.strTaskRunningLog = '正在检查数据库,从未获取的记录' for iRecWhole in self.stockListAll: allItemsByDict[iRecWhole['code']] = iRecWhole['code'] for iRecPart in itemsInZJLX: partItemsByDict[iRecPart['_id']] = iRecPart['_id'] for iRec in allItemsByDict: try: stock_code = partItemsByDict[iRec] except Exception as ee: #part 里面 没有 这个股票代码 needToAppendCode.append(iRec) #print(ee) #print(needToAppendCode) for codeAppend in needToAppendCode: anItem = { '_id': codeAppend, 'first_date': '第一次获取', 'last_date': '第一次获取', 'total_num': 0, 'need_update': 'Yes' } itemsInZJLX.append(anItem) def takeCode(elem): return elem['_id'] itemsInZJLX.sort(key=takeCode) iIndexZJLX = 0 zjlxCount = len(itemsInZJLX) for itemInZjlx in itemsInZJLX: try: iIndexZJLX = iIndexZJLX + 1 self.strTaskRunningLog = '正在检查数据:{} 进度:{}/{}'.format( itemInZjlx, iIndexZJLX, zjlxCount) # item['_id'], item['start_date'], item['end_date'], item['total_num'], item['need_update'] col0 = itemInZjlx['_id'] col1 = itemInZjlx['first_date'] col2 = itemInZjlx['last_date'] col3 = itemInZjlx['total_num'] col4 = itemInZjlx['need_update'] self.pyqtSignalToLogTable.emit(col0, col1, col2, col3, col4) if itemInZjlx['need_update'] == 'Yes': self.zjlxRecNeedUpdateStockCodes.append(col0) except Exception as ee: print(ee) self.strTaskRunningResult = " 任务结束 " pass
def doStart(self): self.bntStop.setEnabled(False) self.bntStart.setEnabled(False) # 🛠todo 关闭上一次启动的服务区进程 for i in range(processNum): try: process_port = str(port_number_start + i) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Connect the socket to the port where the server is listening server_address = ('localhost', int(process_port)) # print('connecting to {} port {}'.format(*server_address)) sock.connect(server_address) strMsg = 'shutdown:shutdown' message = strMsg.encode() sock.sendall(message) # print('sending {!r}'.format(message)) # print('closing socket') #sock.close() except Exception as ee: # print(ee) pass finally: pass # 检查数据库是否已经开启 try: stockListAll = QA_fetch_stock_list() except Exception as ee: #print(ee) strError = ee.__str__() msg = QMessageBox() msg.setIcon(QMessageBox.Information) msg.setText("获取股票代码失败,无法连接数据库!😹") msg.setInformativeText(strError) msg.setWindowTitle("提示:") msg.setDetailedText(": 请确认mongodb 是否运行正常") #retval = msg.exec_() msg.exec() return stockCountAll = len(stockListAll) labelAll = "%d/%d,还剩:%d,进度:%f,剩余小时:%s" % (0, stockCountAll, stockCountAll, 0.0, '未知') self.labelAllProgress.setText(labelAll) # 🛠todo #print("启动服务进程") # xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables/__file__ realPath = os.path.realpath( __file__ ) # xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables/__file__ realDir0 = os.path.dirname(realPath) # xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables realDir1 = os.path.dirname(realDir0) # xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI realDir2 = os.path.dirname(realDir1) for i in range(processNum): process_port = str(port_number_start + i) p = subprocess.Popen([ 'python', './QUANTAXIS_Monitor_GUI/TasksByProcess/SubSeleniumProcess.py', process_port ], cwd=realDir2) #stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #print(p) time.sleep(1) # 尝试连接是否正常工作 # 🛠todo 判断服务进程socket是否已经就绪 time.sleep(30) for i in range(processNum): try: #print("连接服务进程的线程") worker = Worker() worker.logTbl = self.logTbl worker.totalProgressBar = self.progressBarAll worker.totalProgressLabel = self.labelAllProgress worker.process_port = port_number_start + i worker.progressBar = self.subProgressBar[i] worker.progressLabel = self.subProcessLabel[i] worker.progressBar.setMaximum(10000) worker.progressLabel.setText("准备开始。。。") # Worker cannot be converted to PyQt5.QtCore.QObject in this context worker.trigger_new_sub_process_log.connect( worker.handleNewSubProcessLog) worker.trigger_new_sub_process_progress.connect( worker.handleNewSubProcessProgress) #self.threadpool.start(self.worker) #todo fix here 没有用到赞的 self.Thread_List.append(worker) worker.start() time.sleep(1) except Exception as ee: #print(ee) pass self.startup_task_time = datetime.now() self.timer.start(15000) # 设置计时间隔并启动, 更新整体进度 self.progressBarAll.setMaximum(1000000) self.bntStop.setEnabled(True)
def run(self): ''' Your code goes in this function ''' try: stockList = QA_fetch_stock_list() stockCount = len(stockList) subStockList = [] quotient = stockCount // processNum remainder = stockCount % processNum thread_num = self.process_port - port_number_start formStock = 0 toStock = 0 if thread_num < processNum - 1: fromStock = thread_num * quotient + 0 toStock = thread_num * quotient + quotient - 1 for i in range(quotient): aStock = stockList[thread_num * quotient + i] subStockList.append(aStock) else: if remainder != 0: fromStock = thread_num * quotient + 0 toStock = thread_num * quotient + remainder - 1 for i in range(remainder): aStock = stockList[thread_num * quotient + i] subStockList.append(aStock) else: fromStock = thread_num * quotient + 0 toStock = thread_num * quotient + quotient - 1 for i in range(quotient): aStock = stockList[thread_num * quotient + i] subStockList.append(aStock) subStockList.reverse() for i in range(50): if i < len(subStockList[i]): subStockList.remove(subStockList[i]) print("thread_Port%d,一共获取股票%d个, 当前线程分配 %d, from %d to %d" % (self.process_port, stockCount, len(subStockList), fromStock, toStock)) except Exception as ee: print(ee) return for aStock in subStockList: try: fetchCode = aStock['code'] #print("准备获取代码{}".format(fetchCode)) self.fetchCodeZjlx(fetchCode, self.process_port) #print("完成获取代码{}".format(fetchCode)) if mutex.acquire(): #更新总体进度,这个是多线程 ,1/100 , 还剩1 , 1% #labelAll = "%d/%d,还剩:%d,进度:%f" % (0, stockCountAll, stockCountAll, 0.0) txt = self.totalProgressLabel.text() progres_lables = txt.split(',') progress_stock_number = progres_lables[0].split('/') already_got = int(progress_stock_number[0]) total_remain = int(progress_stock_number[1]) already_got = already_got + 1 reamin_stock = total_remain - already_got progress_percent = already_got / total_remain sRemainHour = progres_lables[3].split(':')[1] labelAll = "%d/%d,还剩:%d,进度:%f,剩余小时:%s" % ( already_got, total_remain, reamin_stock, progress_percent, sRemainHour) self.totalProgressLabel.setText(labelAll) mutex.release() except Exception as ee: mutex.release() print(ee) pass return
def re_build(target, positions, sub_accounts, trading_date, percent, exceptions, k=100): sub_accounts = sub_accounts - 10000 positions = positions[positions['股票余额'].astype(float) > 0] positions = positions[positions['股票余额'].astype(float) > 0] positions['上市时间'] = positions['证券代码'].apply( lambda x: date_func(str(QA_fetch_stock_to_market_date(x)))) if exceptions is not None: exceptions = exceptions.extend( list(positions[positions['上市时间'] <= 15].set_index('证券代码').index)) else: exceptions = list( positions[positions['上市时间'] <= 15].set_index('证券代码').index) if target is None: res = pd.concat([ positions.set_index('证券代码'), QA_fetch_stock_fianacial_adv( list(positions.set_index('证券代码').index), trading_date, trading_date).data.reset_index('date')[['NAME', 'INDUSTRY']], QA_fetch_stock_day_adv(list(positions.set_index('证券代码').index), QA_util_get_last_day(trading_date, 60), trading_date).to_qfq().data. loc[trading_date].reset_index('date')['close'] ], axis=1) if exceptions is not None: exceptions_list = [ i for i in list(res.index) if i not in exceptions ] res = res.loc[exceptions_list] else: pass avg_account = 0 res = res.assign(tar=avg_account) res['cnt'] = 0 res['real'] = 0 res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x: float(x)) ).apply(lambda x: math.floor(x / 100) * 100) else: tar1 = target.reset_index().groupby('code').max() tar1['double'] = target.reset_index().groupby('code')['RANK'].count() target = tar1 if exceptions is not None: exceptions_list = [ i for i in list(target.index) if i not in exceptions ] exceptions_listb = [ i for i in list(positions.set_index('证券代码').index) if i not in exceptions ] r1 = target.loc[exceptions_list].join( positions.set_index('证券代码').loc[exceptions_listb], how='outer') else: r1 = target.join(positions.set_index('证券代码'), how='outer') r1['股票余额'] = r1['股票余额'].fillna(0) realtm = QA_fetch_get_stock_realtime( 'tdx', code=[ x for x in list(r1.index) if x in list(QA_fetch_stock_list().index) ]).reset_index('datetime')[[ 'ask1', 'ask_vol1', 'bid1', 'bid_vol1' ]] close = QA_fetch_stock_day_adv( list(r1.index), QA_util_get_last_day(trading_date, 60), trading_date).data.loc[trading_date].reset_index('date')['close'] res = r1.join(QA_fetch_stock_fianacial_adv( list(r1.index), trading_date, trading_date).data.reset_index('date')[['NAME', 'INDUSTRY']], how='left').join(realtm, how='left').join(close, how='left') avg_account = (sub_accounts * percent) / target['double'].sum() res = res.assign(tar=avg_account) res.ix[res['RANK'].isnull(), 'tar'] = 0 res['tar'] = res['tar'] * res['double'] res['amt'] = res.apply(lambda x: func1(x['ask1'], x['bid1']), axis=1) res['cnt'] = (res['tar'] / res['amt'] / 100).apply(lambda x: round(x, 0) * 100) res['real'] = res['cnt'] * res['amt'] res = res.sort_values(by='amt', ascending=False) res = res.fillna(0) res1 = res[res['tar'] > 0] res2 = res[res['tar'] == 0] res1.ix[-1, 'cnt'] = round( (res1['real'][-1] - (res1['real'].sum() - res1['tar'].sum())) / res1['ask1'][-1] / 100, 0) * 100 - k res = pd.concat([res1, res2]) res['real'] = res['cnt'] * res['amt'] res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x: float(x)) ).apply(lambda x: math.floor(x / 100) * 100) return (res)