def QA_SU_crawl_eastmoney(action="zjlx", stockCode=None):
'''
:param action: zjlx 后期支持其他的操作类型
:param stockCode: 股票代码
:return:
'''
stockItems = QA_fetch_stock_list()
if stockCode == "all":
# 读取tushare股票列表代码
print("💪 一共需要获取 %d 个股票的 资金流向 , 需要大概 %d 小时" %
(len(stockItems), (len(stockItems)*5)/60/60))
code_list = []
for stock in stockItems:
code_list.append(stock['code'])
# print(stock['code'])
crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(
code_list)
# print(stock)
return
else:
# todo 检查股票代码是否合法
# return crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(stockCode=stockCode)
code_list = []
code_list.append(stockCode)
return crawl_eastmoney_file.QA_request_eastmoney_zjlx(param_stock_code_list=code_list)
def test_QThread_Check_ZJLX_DB_Count(self):
qthread = QThread_Check_ZJLX_DB_Status()
stockListAll = QA_fetch_stock_list()
for iRec in stockListAll:
stockCode = iRec['code']
lstRet = qthread.QA_count_eastmoney_stock_xjlc_record_count_one_by_one(str_stock_code=stockCode)
print(lstRet)
def QA_fetch_stock_list_adv(collections=DATABASE.stock_list):
'''
'获取股票列表'
:param collections: mongodb 数据库
:return: DataFrame
'''
stock_list_items = QA_fetch_stock_list(collections)
if len(stock_list_items) == 0:
print("QA Error QA_fetch_stock_list_adv call item for item in collections.find() return 0 item, maybe the DATABASE.stock_list is empty!")
return None
return stock_list_items
def QA_fetch_stock_list_adv(collections=DATABASE.stock_list):
'''
'获取股票列表'
:param collections: mongodb 数据库
:return: DataFrame
'''
stock_list_items = QA_fetch_stock_list(collections)
if len(stock_list_items) == 0:
print(
"QA Error QA_fetch_stock_list_adv call item for item in collections.find() return 0 item, maybe the DATABASE.stock_list is empty!"
)
return None
return pd.DataFrame(stock_list_items).drop(
'_id', axis=1, inplace=False).set_index('code', drop=False)
def run(self):
#time.sleep(2)
print("QThread_GetStockList is running")
# 检查数据库是否已经开启
try:
self.stockListAll = QA_fetch_stock_list()
except Exception as ee:
# print(ee)
self.strTaskRunningResult = ee.__str__()
self.stockListAll = None
self.stockCountAll = 0
return
self.stockCountAll = len(self.stockListAll)
self.strTaskRunningResult = "成功获取 股票列表， 🔗 数据库成功 , 共 {} 个股票 😄".format(
self.stockCountAll)
pass
def get(self):
page = int(self.get_argument('page', default=1))
page_size = int(self.get_argument('page_size', default=10))
start = (page - 1) * page_size
end = page * page_size - 1
stock_list = QA_fetch_stock_list()
stock_list['pre_close'] = stock_list['pre_close'].apply(
lambda x: round(x, 2))
stock_fav = QA_fetch_stock_fav()
stock_list = stock_list[stock_list["code"].isin(
stock_fav['code'].tolist())]
total = len(stock_list)
data = QA_util_to_json_from_pandas(stock_list[start:end])
return self.write({
'result': data,
'page': int(page),
'page_size': int(page_size),
'total': int(total)
})
def QA_SU_crawl_eastmoney(action="zjlx",stockCode=None, fromStockCodeToStart=None):
'''
:param action: zjlx 后期支持其他的操作类型
:param stockCode: 股票代码
:return:
'''
stockItems = QA_fetch_stock_list()
if stockCode=="all" or stockCode== "continue":
code_list = []
bCheckpointAfter = False
for stock in stockItems:
if stock['code'] == fromStockCodeToStart:
bCheckpointAfter = True
if fromStockCodeToStart != None and bCheckpointAfter == False and stock['code'] != fromStockCodeToStart:
continue
elif fromStockCodeToStart != None and stock['code'] == fromStockCodeToStart:
bCheckpointAfter = True
code_list.append(stock['code'])
#print(stock['code'])
#读取tushare股票列表代码
print("💪 一共需要获取 %d 个股票的 资金流向 , 需要大概 %d 小时" % (len(code_list), (len(code_list)*30)/60/60 ))
crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(code_list)
#print(stock)
return
else:
#todo 检查股票代码是否合法
#return crawl_eastmoney_file.QA_read_eastmoney_zjlx_web_page_to_sqllite(stockCode=stockCode)
code_list = []
code_list.append(stockCode)
return crawl_eastmoney_file.QA_request_eastmoney_zjlx(param_stock_code_list=code_list)
def QA_statistic_block_data(self):
# 🛠 todo 直接使用monogodb 数据库去统计
try:
block_list_df = QA_fetch_stock_block()
blockNameList = {}
block_list_df = block_list_df.set_index(['blockname'])
for strIndexBlockName in block_list_df.index:
try:
dict = blockNameList[strIndexBlockName]
dict['count'] = dict['count'] + 1
self.strTaskRunningLog = "正在统计板块{} 股票个数{}".format(strIndexBlockName, dict['count'])
except Exception as ee:
dictNew = {}
dictNew['up'] = 0 #上涨家数
dictNew['even'] = 0 #平盘家数
dictNew['down'] = 0 #下跌家数
dictNew['count'] = 0 #板块股票数
dictNew['upRatio'] = 0.0 #上涨家数比率
dictNew['downRation'] = 0.0 #下跌家数比率
blockNameList[strIndexBlockName] = dictNew
#row2 = block_list_df[i]
#######################################################################################################
strToday = QADate.QA_util_today_str()
strEndDate = QADate_trade.QA_util_get_real_date(date = strToday)
strStartDate = QADate_trade.QA_util_get_last_day(date= strEndDate)
codeTwoDayPriceeDict = {}
stock_list = QA_fetch_stock_list()
stock_list_len = len(stock_list)
stock_name_count = 0
for aStockCode in stock_list:
stock_name_count = stock_name_count + 1
strAcode = aStockCode['code']
strAName = aStockCode['name']
priceTowDay = QA_fetch_stock_day_adv(code=strAcode, start=strStartDate, end=strEndDate)
up = None
if priceTowDay is not None:
priceTowDay.to_qfq()
# price_len = len(priceTowDay)
# print(priceTowDay)
price_len = len(priceTowDay)
if price_len == 2:
closeSerial = priceTowDay.close
# todayPrice = priceTowDay[1].close
v1 = closeSerial[0]
v2 = closeSerial[1]
if v1 < v2:
up = True
else:
up = False
codeTwoDayPriceeDict[strAcode] = {'pricetwoday':priceTowDay,
'up':up}
if codeTwoDayPriceeDict[strAcode]['up'] is not None and codeTwoDayPriceeDict[strAcode]['up'] == True:
self.strTaskRunningLog = "正在获取股票价格 {} {} 上涨 ⬆🔺, 进度{}/{} " \
.format(strAcode, strAName, stock_name_count, stock_list_len)
elif codeTwoDayPriceeDict[strAcode]['up'] is not None and codeTwoDayPriceeDict[strAcode]['up'] == False:
self.strTaskRunningLog = "正在获取股票价格 {} {} 下跌 ⬇️, 进度{}/{} " \
.format(strAcode, strAName, stock_name_count, stock_list_len)
else:
self.strTaskRunningLog = "正在获取股票价格 {} {} 和前一天的无法统计️, 进度{}/{} " \
.format(strAcode, strAName, stock_name_count, stock_list_len)
##########################################################################################################
len_block_size = len(blockNameList)
count = 0
for blockName in blockNameList.keys():
count = count + 1
stocks_in_block = QA_fetch_stock_block_adv(code=None, blockname= blockName)
self.strTaskRunningLog = "正在统计板块{} ,进度{}/{}".format(blockName,count,len_block_size)
code_list = stocks_in_block.code
iCodeLength = len(code_list)
iCountForCode = 0
for iCode in code_list:
iCountForCode = iCountForCode + 1
try:
priceTowDay = codeTwoDayPriceeDict[iCode]['pricetwoday']
upValue = codeTwoDayPriceeDict[iCode]['up']
if priceTowDay is not None:
if upValue == True:
blockNameList[blockName]['up'] = blockNameList[blockName]['up'] + 1
elif upValue == False:
blockNameList[blockName]['down'] = blockNameList[blockName]['down'] + 1
except Exception as ee:
strErrorMsg = ee.__str__()
print(strErrorMsg)
self.strTaskRunningLog = "正在统计板块{} ,进度{}/{} \n 股票 {}, 进度 {}/{}"\
.format(blockName, count, len_block_size, iCode, iCountForCode, iCodeLength)
pass
#print(stocks_in_block)
#######################################################################################################
except Exception as eee:
strErrorMsg = eee.__str__()
print(strErrorMsg)
finally:
return blockNameList
def run(self):
#
def find(f, seq):
"""Return first item in sequence where f(item) == True."""
for item in seq:
if f(item):
return item
self.zjlxRecNeedUpdateStockCodes.clear()
self.stockListAll.clear()
self.stockListAll = QA_fetch_stock_list()
self.strTaskRunningLog = '正在检查数据库，已经获取的数据'
itemsInZJLX = self.QA_count_eastmoney_stock_xjlc_record_count_by_aggregate(
)
'''
追究没有获取过的记录，
'''
# 非常慢
# codeNeedAppend = []
# for aItemInAllStock in self.stockListAll:
# code0 = aItemInAllStock['code']
# #self.strTaskRunningLog = '正在检查数据 {}'.format(code0)
#
# foundItem = find(lambda aRec: aRec['_id'] == code0, itemsInZJLX)
# if foundItem is None:
# codeNeedAppend.append(code0)
#
#
# print(codeNeedAppend)
# 使用 dict
allItemsByDict = {}
partItemsByDict = {}
needToAppendCode = []
self.strTaskRunningLog = '正在检查数据库，从未获取的记录'
for iRecWhole in self.stockListAll:
allItemsByDict[iRecWhole['code']] = iRecWhole['code']
for iRecPart in itemsInZJLX:
partItemsByDict[iRecPart['_id']] = iRecPart['_id']
for iRec in allItemsByDict:
try:
stock_code = partItemsByDict[iRec]
except Exception as ee:
#part 里面 没有 这个股票代码
needToAppendCode.append(iRec)
#print(ee)
#print(needToAppendCode)
for codeAppend in needToAppendCode:
anItem = {
'_id': codeAppend,
'first_date': '第一次获取',
'last_date': '第一次获取',
'total_num': 0,
'need_update': 'Yes'
}
itemsInZJLX.append(anItem)
def takeCode(elem):
return elem['_id']
itemsInZJLX.sort(key=takeCode)
iIndexZJLX = 0
zjlxCount = len(itemsInZJLX)
for itemInZjlx in itemsInZJLX:
try:
iIndexZJLX = iIndexZJLX + 1
self.strTaskRunningLog = '正在检查数据:{} 进度:{}/{}'.format(
itemInZjlx, iIndexZJLX, zjlxCount)
# item['_id'], item['start_date'], item['end_date'], item['total_num'], item['need_update']
col0 = itemInZjlx['_id']
col1 = itemInZjlx['first_date']
col2 = itemInZjlx['last_date']
col3 = itemInZjlx['total_num']
col4 = itemInZjlx['need_update']
self.pyqtSignalToLogTable.emit(col0, col1, col2, col3, col4)
if itemInZjlx['need_update'] == 'Yes':
self.zjlxRecNeedUpdateStockCodes.append(col0)
except Exception as ee:
print(ee)
self.strTaskRunningResult = " 任务结束 "
pass
def doStart(self):
self.bntStop.setEnabled(False)
self.bntStart.setEnabled(False)
# 🛠todo 关闭上一次启动的服务区进程
for i in range(processNum):
try:
process_port = str(port_number_start + i)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
server_address = ('localhost', int(process_port))
# print('connecting to {} port {}'.format(*server_address))
sock.connect(server_address)
strMsg = 'shutdown:shutdown'
message = strMsg.encode()
sock.sendall(message)
# print('sending {!r}'.format(message))
# print('closing socket')
#sock.close()
except Exception as ee:
# print(ee)
pass
finally:
pass
# 检查数据库是否已经开启
try:
stockListAll = QA_fetch_stock_list()
except Exception as ee:
#print(ee)
strError = ee.__str__()
msg = QMessageBox()
msg.setIcon(QMessageBox.Information)
msg.setText("获取股票代码失败，无法连接数据库！😹")
msg.setInformativeText(strError)
msg.setWindowTitle("提示：")
msg.setDetailedText(": 请确认mongodb 是否运行正常")
#retval = msg.exec_()
msg.exec()
return
stockCountAll = len(stockListAll)
labelAll = "%d/%d,还剩:%d,进度:%f,剩余小时:%s" % (0, stockCountAll,
stockCountAll, 0.0, '未知')
self.labelAllProgress.setText(labelAll)
# 🛠todo
#print("启动服务进程")
# xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables/__file__
realPath = os.path.realpath(
__file__
) # xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables/__file__
realDir0 = os.path.dirname(realPath)
# xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI/MainTables
realDir1 = os.path.dirname(realDir0)
# xxxx/QUANTAXIS/QUANTAXIS_Monitor_GUI
realDir2 = os.path.dirname(realDir1)
for i in range(processNum):
process_port = str(port_number_start + i)
p = subprocess.Popen([
'python',
'./QUANTAXIS_Monitor_GUI/TasksByProcess/SubSeleniumProcess.py',
process_port
],
cwd=realDir2)
#stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#print(p)
time.sleep(1)
# 尝试连接是否正常工作
# 🛠todo 判断服务进程socket是否已经就绪
time.sleep(30)
for i in range(processNum):
try:
#print("连接服务进程的线程")
worker = Worker()
worker.logTbl = self.logTbl
worker.totalProgressBar = self.progressBarAll
worker.totalProgressLabel = self.labelAllProgress
worker.process_port = port_number_start + i
worker.progressBar = self.subProgressBar[i]
worker.progressLabel = self.subProcessLabel[i]
worker.progressBar.setMaximum(10000)
worker.progressLabel.setText("准备开始。。。")
# Worker cannot be converted to PyQt5.QtCore.QObject in this context
worker.trigger_new_sub_process_log.connect(
worker.handleNewSubProcessLog)
worker.trigger_new_sub_process_progress.connect(
worker.handleNewSubProcessProgress)
#self.threadpool.start(self.worker)
#todo fix here 没有用到赞的
self.Thread_List.append(worker)
worker.start()
time.sleep(1)
except Exception as ee:
#print(ee)
pass
self.startup_task_time = datetime.now()
self.timer.start(15000) # 设置计时间隔并启动, 更新整体进度
self.progressBarAll.setMaximum(1000000)
self.bntStop.setEnabled(True)
def run(self):
'''
Your code goes in this function
'''
try:
stockList = QA_fetch_stock_list()
stockCount = len(stockList)
subStockList = []
quotient = stockCount // processNum
remainder = stockCount % processNum
thread_num = self.process_port - port_number_start
formStock = 0
toStock = 0
if thread_num < processNum - 1:
fromStock = thread_num * quotient + 0
toStock = thread_num * quotient + quotient - 1
for i in range(quotient):
aStock = stockList[thread_num * quotient + i]
subStockList.append(aStock)
else:
if remainder != 0:
fromStock = thread_num * quotient + 0
toStock = thread_num * quotient + remainder - 1
for i in range(remainder):
aStock = stockList[thread_num * quotient + i]
subStockList.append(aStock)
else:
fromStock = thread_num * quotient + 0
toStock = thread_num * quotient + quotient - 1
for i in range(quotient):
aStock = stockList[thread_num * quotient + i]
subStockList.append(aStock)
subStockList.reverse()
for i in range(50):
if i < len(subStockList[i]):
subStockList.remove(subStockList[i])
print("thread_Port%d,一共获取股票%d个, 当前线程分配 %d， from %d to %d" %
(self.process_port, stockCount, len(subStockList), fromStock,
toStock))
except Exception as ee:
print(ee)
return
for aStock in subStockList:
try:
fetchCode = aStock['code']
#print("准备获取代码{}".format(fetchCode))
self.fetchCodeZjlx(fetchCode, self.process_port)
#print("完成获取代码{}".format(fetchCode))
if mutex.acquire():
#更新总体进度，这个是多线程 ,1/100 , 还剩1 , 1%
#labelAll = "%d/%d,还剩:%d,进度:%f" % (0, stockCountAll, stockCountAll, 0.0)
txt = self.totalProgressLabel.text()
progres_lables = txt.split(',')
progress_stock_number = progres_lables[0].split('/')
already_got = int(progress_stock_number[0])
total_remain = int(progress_stock_number[1])
already_got = already_got + 1
reamin_stock = total_remain - already_got
progress_percent = already_got / total_remain
sRemainHour = progres_lables[3].split(':')[1]
labelAll = "%d/%d,还剩:%d,进度:%f,剩余小时:%s" % (
already_got, total_remain, reamin_stock,
progress_percent, sRemainHour)
self.totalProgressLabel.setText(labelAll)
mutex.release()
except Exception as ee:
mutex.release()
print(ee)
pass
return
def re_build(target,
positions,
sub_accounts,
trading_date,
percent,
exceptions,
k=100):
sub_accounts = sub_accounts - 10000
positions = positions[positions['股票余额'].astype(float) > 0]
positions = positions[positions['股票余额'].astype(float) > 0]
positions['上市时间'] = positions['证券代码'].apply(
lambda x: date_func(str(QA_fetch_stock_to_market_date(x))))
if exceptions is not None:
exceptions = exceptions.extend(
list(positions[positions['上市时间'] <= 15].set_index('证券代码').index))
else:
exceptions = list(
positions[positions['上市时间'] <= 15].set_index('证券代码').index)
if target is None:
res = pd.concat([
positions.set_index('证券代码'),
QA_fetch_stock_fianacial_adv(
list(positions.set_index('证券代码').index), trading_date,
trading_date).data.reset_index('date')[['NAME', 'INDUSTRY']],
QA_fetch_stock_day_adv(list(positions.set_index('证券代码').index),
QA_util_get_last_day(trading_date, 60),
trading_date).to_qfq().data.
loc[trading_date].reset_index('date')['close']
],
axis=1)
if exceptions is not None:
exceptions_list = [
i for i in list(res.index) if i not in exceptions
]
res = res.loc[exceptions_list]
else:
pass
avg_account = 0
res = res.assign(tar=avg_account)
res['cnt'] = 0
res['real'] = 0
res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x: float(x))
).apply(lambda x: math.floor(x / 100) * 100)
else:
tar1 = target.reset_index().groupby('code').max()
tar1['double'] = target.reset_index().groupby('code')['RANK'].count()
target = tar1
if exceptions is not None:
exceptions_list = [
i for i in list(target.index) if i not in exceptions
]
exceptions_listb = [
i for i in list(positions.set_index('证券代码').index)
if i not in exceptions
]
r1 = target.loc[exceptions_list].join(
positions.set_index('证券代码').loc[exceptions_listb], how='outer')
else:
r1 = target.join(positions.set_index('证券代码'), how='outer')
r1['股票余额'] = r1['股票余额'].fillna(0)
realtm = QA_fetch_get_stock_realtime(
'tdx',
code=[
x for x in list(r1.index)
if x in list(QA_fetch_stock_list().index)
]).reset_index('datetime')[[
'ask1', 'ask_vol1', 'bid1', 'bid_vol1'
]]
close = QA_fetch_stock_day_adv(
list(r1.index), QA_util_get_last_day(trading_date, 60),
trading_date).data.loc[trading_date].reset_index('date')['close']
res = r1.join(QA_fetch_stock_fianacial_adv(
list(r1.index), trading_date,
trading_date).data.reset_index('date')[['NAME', 'INDUSTRY']],
how='left').join(realtm, how='left').join(close,
how='left')
avg_account = (sub_accounts * percent) / target['double'].sum()
res = res.assign(tar=avg_account)
res.ix[res['RANK'].isnull(), 'tar'] = 0
res['tar'] = res['tar'] * res['double']
res['amt'] = res.apply(lambda x: func1(x['ask1'], x['bid1']), axis=1)
res['cnt'] = (res['tar'] / res['amt'] /
100).apply(lambda x: round(x, 0) * 100)
res['real'] = res['cnt'] * res['amt']
res = res.sort_values(by='amt', ascending=False)
res = res.fillna(0)
res1 = res[res['tar'] > 0]
res2 = res[res['tar'] == 0]
res1.ix[-1, 'cnt'] = round(
(res1['real'][-1] - (res1['real'].sum() - res1['tar'].sum())) /
res1['ask1'][-1] / 100, 0) * 100 - k
res = pd.concat([res1, res2])
res['real'] = res['cnt'] * res['amt']
res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x: float(x))
).apply(lambda x: math.floor(x / 100) * 100)
return (res)
