def init(forceUpdate):
def cb(**kw):
df = ts.get_today_all()
df['pick'] = 0
return df
df_todayAll = Utils.queryData('today_all','code',engine, cb, forceUpdate=forceUpdate, sql='select * from today_all where pick = 1', load_if_empty=False)
strTime = time.strftime('%H:%M:%S',time.localtime(time.time()))
while strTime < '09:30:01':
time.sleep(0.1)
strTime = time.strftime('%H:%M:%S',time.localtime(time.time()))
step = 880
start = 0
codeList = []
length = len(df_todayAll)
if length == 0:
MyLog.info('no stocks to calc')
return
origin_code_list = df_todayAll['code'].tolist()
while start < length:
end = start + step
if end >= length:
end = length
df_temp = df_todayAll.iloc[start:end]
df = ts.get_realtime_quotes(df_temp['code'].tolist())
df = df[df.apply(analyze.isOpenMatch, axis=1)]
for code in df['code'].tolist():
codeList.append(code)
start = end
return (origin_code_list, codeList)
def start_test(codes):
for code in codes:
try:
src_datas[code] = pd.read_sql_table('live_' + code, con=engine)
except Exception as e:
MyLog.error('read mock data error \n')
MyLog.error(str(e) + '\n')
for i in range(5200):
run(i)
input('please enter to exit')
def start_test_by_df(df_list):
for df in df_list:
try:
code = df.iloc[0]['code']
src_datas[code] = df
except Exception as e:
MyLog.error('read mock data error \n')
MyLog.error(str(e) + '\n')
for i in range(5200):
run(i)
return result
def addDataHandler(self, row):
code = row['code']
if code in self.get_buyed():
return
if code in self.__data and self.__data[code].len() > 0:
self.__data[code].add_Line(row)
else:
self.__data[code] = Stock(code, row)
if float(row['pre_close']) != 0 and (float(row['price']) - float(
row['pre_close'])) / float(row['pre_close']) * 100 >= 9.9:
if not (code in self.get_buyed()):
MyLog.info('[%s] reach 10' % code)
self.add_buyed(code, True)
def recoverData(self, codes):
for code in codes:
try:
src_data = pd.read_sql_table('live_' + code, con=self.__engine)
if src_data is not None and len(src_data) > 0:
last = src_data.iloc[-1]
now_date = time.strftime('%Y-%m-%d',
time.localtime(time.time()))
if now_date == last['date']:
self.__data[code] = Stock(code, src_data)
except Exception as e:
MyLog.error('recover data error \n')
MyLog.error(str(e) + '\n')
def eyes_on_codes_change():
global old_code_list
global num_splits
def cb(**kw):
df = ts.get_today_all()
df['pick'] = 0
return df
new_df = Utils.queryData('today_all','code',engine, cb, forceUpdate=False, sql='select * from today_all where pick = 1', load_if_empty=False)
if len(new_df) == 0:
return
new_code_list = new_df['code'].tolist()
old_code_list.sort()
new_code_list.sort()
add = []
delete = []
for n_code in new_code_list:
if n_code not in old_code_list:
add.append(n_code)
for o_code in old_code_list:
if o_code not in new_code_list:
delete.append(o_code)
old_code_list = new_code_list
for code in delete:
for key in codeSplitMaps:
if code in codeSplitMaps[key]:
codeSplitMaps[key].remove(code)
break
length = len(add)
if length == 0:
return
x = length // 100
y = length % 100
size = x + (1 if y != 0 else 0)
begin = 0
step = setting.get_t1()['split_size']
for i in range(size):
end = begin + step
if end > length:
end = length
code_list = add[begin:end]
codeSplitMaps[num_splits] = code_list
queue = manager.Queue()
queueMaps[num_splits] = queue
pool.apply_async(run, args=(queue,balance,lock))
MyLog.info('calc new added stocks %s' % code_list)
num_splits = num_splits + 1
begin = end
if begin >= length:
break
proxyManager.append_proxy(size)
def addDataHandler(self, row):
code = row['code']
if code in self.get_buyed() or code in self.get_ignore():
return
if code in self.__data and self.__data[code].len() > 0:
self.__data[code].add_Line(row)
else:
self.__data[code] = NewStock(code, row)
if float(row['pre_close']) != 0 and (float(row['price']) - float(
row['pre_close'])) / float(row['pre_close']) * 100 >= 9.9:
MyLog.info('[%s] reach 10' % code)
save = False
if float(row['price']) != float(row['open']):
save = True
self.add_ignore(code, save=save)
def run(queue,balance,lock):
MyLog.info('child process %s is running' % os.getpid())
try:
dh = None
data = queue.get(True)
while data is not None and data['df'] is not None and len(data['df']) > 0:
timestamp = data['timestamp']
df = data['df']
zs = data['zs']
if data['balance'] is not None:
balance.value = data['balance']
if dh is None:
dh = NewDataHolder2()
dh.addData(df)
analyze.calcMain(zs,dh,timestamp,balance,lock)
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
def run(queue,queueout):
MyLog.info('child process %s is running' % os.getpid())
try:
dh = None
data = queue.get(True)
while data is not None and data['df'] is not None and len(data['df']) > 0:
timestamp = data['timestamp']
df = data['df']
zs = data['zs']
buyCount = data['buyCount']
if dh is None:
dh = NewDataHolder()
dh.addData(df)
res = analyze.calcMain(zs,dh,timestamp,buyCount)
if len(res) > 0:
for code in res:
dh.add_buyed(code)
queueout.put({'buyed' : 1})
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
def run(queue):
MyLog.info('child process %s is running' % os.getpid())
try:
dh = None
data = queue.get(True)
while data is not None and data['df'] is not None and len(
data['df']) > 0:
timestamp = data['timestamp']
zs = data['zs']
df = data['df']
hygn = None
netMoney = None
s = int(round(time.time() * 1000))
if dh is None:
codeList = df['code'].tolist()
dh = DataHolder(codeList)
dh.addData(df)
res = analyze.calcMain(zs, dh, hygn, netMoney, timestamp)
if len(res) > 0:
for code in res:
dh.add_buyed(code, True)
MyLog.debug('process %s, calc data time = %d' %
(os.getpid(), (int(round(time.time() * 1000)) - s)))
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
def run(codes, dh):
# if len(dh.get_buyed()) > 0:
# for code in dh.get_buyed():
# if code in codes:
# codes.remove(code)
try:
# if len(codes) > 0:
# df = ts.get_realtime_quotes(codes)
# dh.addData(df)
# res = analyze.calcMain(dh)
# if res != '':
# dh.add_buyed(res)
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
print(codes)
except Exception as e:
MyLog.error('get data error %s %s' % (codes, str(e)))
finally:
global timer
timer = threading.Timer(setting.get_t1()['get_data_inter'],
run,
args=[codes, dh])
timer.start()
def saveData(self, data):
try:
line = data.iloc[0]
code = line['code']
data.to_sql('live_' + code,
con=self.__engine,
if_exists='replace',
index=False)
MyLog.info('[%s] save data' % code)
except Exception as e:
MyLog.error('[%s %s] save [%s] data error \n' %
(line['date'], line['time'], code))
MyLog.error(str(e) + '\n')
def run(queue):
MyLog.info('child process %s is running' % os.getpid())
try:
dh = None
data = queue.get(True)
while data is not None and data['df'] is not None and len(
data['df']) > 0:
df = data['df']
zs = data['zs']
s = int(round(time.time() * 1000))
if dh is None:
codeList = df['code'].tolist()
dh = DataHolder(codeList)
dh.addSellData(df)
analyze.calcMain(zs, dh)
MyLog.debug('process %s, calc data time = %d' %
(os.getpid(), (int(round(time.time() * 1000)) - s)))
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
# thsgn = pd.read_sql_table('concept', con=engine)
# analyze = Analyze(thshy,thsgn)
analyze = Analyze(None,None)
mockTrade = MockTrade()
mockTrade.relogin()
# concept = Concept()
# netMoney = NetMoney()
# hygn = concept.getCurrentTopHYandConcept()
# net = netMoney.getNetMoneyRatio()
zs = ts.get_realtime_quotes(['sh','sz','hs300','sz50','zxb','cyb'])
for code in codes:
try:
src_datas[code] = pd.read_sql_table('live_' + code, con=engine)
except Exception as e:
MyLog.error('read mock data error \n')
MyLog.error(str(e) + '\n')
def run(i):
df = pd.DataFrame()
for code in src_datas:
if i < len(src_datas[code]):
df = df.append(src_datas[code].iloc[i])
if len(df) > 0:
dh.addData(df)
codes = analyze.calcMain(zs,dh,None,None,dt.datetime.now())
if len(codes) > 0:
for code in codes:
dh.add_buyed(code,False)
for i in range(5200):
def getData():
timestamp = dt.datetime.now()
if setting.get_t1()['trade']['enableMock']:
delSeconds = (timestamp - interDataHolder['currentTime']).seconds
delQueryBuySeconds = (timestamp - interDataHolder['queryBuyTime']).seconds
if delSeconds > 30:
interDataHolder['currentTime'] = timestamp
interDataHolder['zs'] = proxyManager.get_realtime_quotes(['sh','sz','hs300','sz50','zxb','cyb'],batch_size=0,use_proxy_no_batch=True)
if delQueryBuySeconds > 120:
interDataHolder['queryBuyTime'] = timestamp
mockTrade.relogin()
count = mockTrade.queryBuyStocks()
if count >= setting.get_t1()['trade']['max_buyed']:
mockTrade.cancelAllBuy()
MyLog.info('buyed 3 stocks')
interDataHolder['stopBuy'] = True
if setting.get_t1()['trade']['enable']:
delSeconds = (timestamp - interDataHolder['currentTime']).seconds
delBalanceSeconds = (timestamp - interDataHolder['balanceTime']).seconds
delQueryBuySeconds = (timestamp - interDataHolder['queryBuyTime']).seconds
if delSeconds > 30:
interDataHolder['currentTime'] = timestamp
interDataHolder['zs'] = proxyManager.get_realtime_quotes(['sh','sz','hs300','sz50','zxb','cyb'],batch_size=0,use_proxy_no_batch=True)
if delBalanceSeconds > 320:
interDataHolder['balanceTime'] = timestamp
try:
lock.acquire()
trade.refresh()
balance = trade.queryBalance()
if balance is not None:
interDataHolder['balance'] = balance
except Exception as e:
interDataHolder['balance'] = None
finally:
lock.release()
if delQueryBuySeconds > 350:
interDataHolder['queryBuyTime'] = timestamp
try:
lock.acquire()
count = trade.queryBuyStocks()
if count >= setting.get_t1()['trade']['max_buyed']:
trade.cancel(None,True)
MyLog.info('buyed 3 stocks')
interDataHolder['stopBuy'] = True
except Exception as e:
pass
finally:
lock.release()
if interDataHolder['stopBuy']:
os._exit(0)
return
strTime = time.strftime('%H:%M:%S',time.localtime(time.time()))
if strTime > '11:30:30' and strTime < '12:59:30':
return
if strTime > '15:30:30':
os._exit(0)
return
if interDataHolder['zs'] is None:
interDataHolder['zs'] = proxyManager.get_realtime_quotes(['sh','sz','hs300','sz50','zxb','cyb'],batch_size=0,use_proxy_no_batch=True)
for key in codeSplitMaps:
df = proxyManager.get_realtime_quotes(codeSplitMaps[key],queueMaps[key],{'timestamp' : timestamp,'df' : None,'zs' : interDataHolder['zs'], 'balance' : interDataHolder['balance']},batch_size=100,async_exe=put_data_to_queue)
if dh is None:
dh = NewDataHolder()
dh.addData(df)
res = analyze.calcMain(zs,dh,timestamp,buyCount)
if len(res) > 0:
for code in res:
dh.add_buyed(code)
queueout.put({'buyed' : 1})
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
if __name__ == '__main__':
MyLog.info('main process %s.' % os.getpid())
def init(forceUpdate):
def cb(**kw):
return ts.get_today_all()
df_todayAll = Utils.queryData('today_all','code',engine, cb, forceUpdate=forceUpdate)
strTime = time.strftime('%H:%M:%S',time.localtime(time.time()))
while strTime < '09:30:01':
time.sleep(0.1)
strTime = time.strftime('%H:%M:%S',time.localtime(time.time()))
step = setting.get_t1()['split_size']
start = 0
codeList = []
length = len(df_todayAll)
while start < length:
end = start + step
zs = data['zs']
s = int(round(time.time() * 1000))
if dh is None:
codeList = df['code'].tolist()
dh = DataHolder(codeList)
dh.addSellData(df)
analyze.calcMain(zs, dh)
MyLog.debug('process %s, calc data time = %d' %
(os.getpid(), (int(round(time.time() * 1000)) - s)))
data = queue.get(True)
except Exception as e:
MyLog.error('error %s' % str(e))
if __name__ == '__main__':
MyLog.info('main process %s.' % os.getpid())
strTime = time.strftime('%H:%M:%S', time.localtime(time.time()))
while strTime < '09:30:01':
time.sleep(0.1)
strTime = time.strftime('%H:%M:%S', time.localtime(time.time()))
if setting.get_t1()['trade']['enableMock']:
mockTrade.relogin()
pool = mp.Pool(1)
manager = mp.Manager()
queue = manager.Queue()
pool.apply_async(run, (queue, ))
sched = BlockingScheduler()
interDataHolder = {'currentTime': dt.datetime.now()}
@sched.scheduled_job('interval',
seconds=setting.get_t1()['get_data_inter'])
def filter(df_todayAll, setting, engine):
result = []
now = dt.datetime.now()
todayStr = now.strftime('%Y-%m-%d')
timeDelta = dt.timedelta(setting.get_longPeriod())
threeMbefore = (now - timeDelta).strftime('%Y-%m-%d')
sm = StrategyManager()
stockManager = StocksManager(engine, setting)
length = len(df_todayAll)
begin = 0
threads = []
for i in range(11):
if length < 10:
end = length
else:
end = begin + length // 10
if end >= length - 1:
end = length
t = threading.Thread(target=subProcessTask,
args=(df_todayAll[begin:end], result,
threeMbefore, sm, stockManager, engine,
setting, todayStr))
t.setDaemon(True)
t.start()
print('start thread filter data %d, %d' % (begin, end))
threads.append(t)
begin = end
if begin >= length:
break
for t in threads:
t.join()
# for index,row in df_todayAll.iterrows():
# code = row['code']
# def cb(**kw):
# return ts.get_k_data(kw['kw']['code'],start=kw['kw']['start'])
# df_3m = Utils.queryData('k_data_' + code,'code',engine, cb, forceUpdate=setting.get_updateToday(),code=code,start=threeMbefore)
# if df_3m is None:
# continue
# # 数据少于180天
# if df_3m.empty or len(df_3m) < setting.get_trendPeriod():
# continue
# #添加最后一行
# if df_3m.iloc[-1].get('date') != todayStr:
# today_df = pd.DataFrame([[todayStr, row['open'],row['trade'],row['high'],row['low'],row['volume']/100,code]],columns=list(['date','open','close','high','low','volume','code']))
# df_3m = df_3m.append(today_df,ignore_index=True)
# # df_3m = df_3m[::-1]
# #计算指标使用半年D数据
# Utils.macd(df_3m)
# Utils.myKdj(df_3m)
# ######################################开始配置计算###########################################
# sm = StrategyManager()
# data = {'df_3m' : df_3m,'df_realTime' : row, 'engine' : engine}
# if sm.start(code, setting.get_Strategy(), data, setting):
# result.append(code)
# df_res = stockManager.buildDataFrame()
# i = 0
# for index,row in df_res.iterrows():
# result.append(index)
# i = i + 1
print('Topsis 综合测评结果排序 : ', result)
MyLog.info('Topsis 综合测评结果排序 : %s' % result)
return result