def objective():
# reading gt & logs
logfile = readlogsFiles()[0]
tmpA = os.path.basename(logfile).split('_')[:3]
tmpB = os.path.basename(logfile).split('_')[3:6]
tmpA = np.array(tmpA).astype(int)
tmpB = np.array(tmpB).astype(int)
AB = tmpA - tmpB
print(f'nk:{AB[0]} tnk:{AB[1]} tk:{AB[2]}')
# OK a-b<0
if all(AB<0):
# U:[None,10], B:[3,]
U,B = myData.loadABLV(logfile)
deltaU = myData.convV2YearlyData(U)
# one : one (※MSE) mode==4(normal) mode==5(reverse)
maxSim = myData.MinErrorNankai(gt, deltaU, mode=4)
else:
# out value
maxSim = 100000000
maxSim = 1/maxSim
# Delate logfile
os.remove(logfile)
return maxSim
def singal(pre_date_seq, para_min):
threshold = 1.0001
# 建立数据库连接,清空相关表
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='admin',
db='stock',
charset='utf8')
cursor = db.cursor()
dc = DC.data_collect2(pre_date_seq[0], date_seq[-1], para_min, threshold)
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
if dc.cnt_pos == 0:
print('No Positive Samples!')
return 0
w = (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
print('No Negtive Samples!')
return 0
# model = svm.SVC(class_weight={1: w})
model = svm.SVC()
model.fit(train, target)
# print(model.score(train,target))
ans2 = model.predict(test_case)
return int(ans2[0])
def svm_predict(stock_code,end_dt):
model_predict_new_start = (datetime.datetime.strptime(end_dt, '%Y-%m-%d') - datetime.timedelta(days=365)).strftime(
'%Y-%m-%d')
dc = DC.data_collect2(stock_code, model_predict_new_start, end_dt)
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
w = 5 * (len(target) / dc.cnt_pos)
model = svm.SVC(class_weight={1: w})
model.fit(train, target)
ans2 = model.predict(test_case)
return ans2[0]
def svm_test(stock,end_dt):
# 清空内存表
db = pymysql.connect(host='127.0.0.1', user='******', passwd='admin', db='stock', charset='utf8')
cursor = db.cursor()
sql_truncate_model_test = 'truncate table model_test'
cursor.execute(sql_truncate_model_test)
db.commit()
model_test_date_start = (datetime.datetime.strptime(end_dt, '%Y-%m-%d') - datetime.timedelta(days=90)).strftime(
'%Y-%m-%d')
sql_model_test_date_seq = "select prevTradeDate from stock_tradecalall a where a.calendarDate >= '%s' and a.calendarDate <= '%s' and a.exchangeCD = 'XSHG' and isOpen = 1 order by a.prevTradeDate asc" % (
model_test_date_start, end_dt)
cursor.execute(sql_model_test_date_seq)
done_set_date_seq = cursor.fetchall()
model_test_date_seq = [x[0] for x in done_set_date_seq]
return_flag = 0
for d in range(len(model_test_date_seq)):
model_test_new_start = (
datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d') - datetime.timedelta(days=365)).strftime(
'%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
try:
dc = DC.data_collect2(stock, model_test_new_start, model_test_new_end)
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
if dc.cnt_pos == 0:
return_flag = 1
break
w = 5 * (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
return_flag = 1
break
model = svm.SVC(class_weight={1: w})
model.fit(train, target)
ans2 = model.predict(test_case)
# print(end_dt + ' ANS : ' + str(ans2[0]) + 'POS_CNT : ' + str(len([x for x in target if x == 1.00])) + ' of total : ' + str(len(target)) + ' start_dt : ' + str(model_test_new_start) + ' end_dt : ' + str(model_test_new_end))
sql_insert = "insert into model_test(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (model_test_new_end, stock, float(ans2[0]))
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
return 0,0,0,0
acc, recall, acc_neg, f1 = ModelEvaluate.ModelEvaluate(stock, model_test_date_seq)
return acc,recall,acc_neg,f1
def encoding(bs, width, heigth):
DCY, DCU, DCV, ACY, ACU, ACV = Compress.encoding(bs, height, width)
YBlocks = DC.DPCM2(DCY)
UBlocks = DC.DPCM2(DCU)
VBlocks = DC.DPCM2(DCV)
for i in range(len(YBlocks)):
AC.Z2Tab(ACY[i], YBlocks[i])
YBlocks[i] = Quantization.reY(YBlocks[i])
YBlocks[i] = DCT.IDCT(YBlocks[i])
for i in range(len(UBlocks)):
AC.Z2Tab(ACU[i], UBlocks[i])
UBlocks[i] = Quantization.reUV(UBlocks[i])
UBlocks[i] = DCT.IDCT(UBlocks[i])
for i in range(len(VBlocks)):
AC.Z2Tab(ACV[i], VBlocks[i])
VBlocks[i] = Quantization.reUV(VBlocks[i])
VBlocks[i] = DCT.IDCT(VBlocks[i])
Y, U, V = DCT.merge(YBlocks, UBlocks, VBlocks, height, width)
img = RGB2YUV.yuv2rgb(Y, U, V, width, height)
cv2.imwrite(r'../photograph_JPEG.jpeg', img)
cv2.imshow("img after encoding", img)
cv2.waitKey(0)
cv2.destroyAllWindows()
from sklearn import svm
import DC
if __name__ == '__main__':
stock = 'AAPL' # 以AAPL为例子
dc = DC.data_collect(stock, '2017-01-01', '2019-08-23')
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 输出对2019-08-14的涨跌预测，1表示涨，-1表示不涨。
print('Forecast:', ans2[0])
def compress(path):
img = cv2.imread(path)
height = img.shape[0]
width = img.shape[1]
Y, U, V = RGB2YUV.rgb2yuv(img, img.shape[1], img.shape[0])
Y = DCT.fill(Y)
U = DCT.fill(U)
V = DCT.fill(V)
blocksY = DCT.split(Y)
blocksU = DCT.split(U)
blocksV = DCT.split(V)
FDCT = []
Quan = []
Z = []
ACnum = []
for block in blocksY:
FDCT.append(DCT.FDCT(block))
Quan.append(Quantization.quanY(FDCT[-1]))
Z.append(AC.ZScan(Quan[-1]))
ACnum.append(AC.RLC(Z[-1]))
DCnum = DC.DPCM(Quan)
#print('Y: ')
Bstr0 = ''
for i in range(len(ACnum)):
Bstr0 += Compress.AllCompressY(DCnum[i], ACnum[i])
#print(Bstr0)
#print(len(Bstr0))
FDCT = []
Quan = []
Z = []
ACnum = []
for block in blocksU:
FDCT.append(DCT.FDCT(block))
Quan.append(Quantization.quanUV(FDCT[-1]))
Z.append(AC.ZScan(Quan[-1]))
ACnum.append(AC.RLC(Z[-1]))
DCnum = DC.DPCM(Quan)
#print('U: ')
Bstr1 = ''
for i in range(len(ACnum)):
Bstr1 += Compress.AllCompressUV(DCnum[i], ACnum[i])
#print(Bstr1)
#print(len(Bstr1))
FDCT = []
Quan = []
Z = []
ACnum = []
for block in blocksV:
FDCT.append(DCT.FDCT(block))
Quan.append(Quantization.quanUV(FDCT[-1]))
Z.append(AC.ZScan(Quan[-1]))
ACnum.append(AC.RLC(Z[-1]))
DCnum = DC.DPCM(Quan)
#print('V: ')
Bstr2 = ''
for i in range(len(ACnum)):
Bstr2 += Compress.AllCompressUV(DCnum[i], ACnum[i])
#print(Bstr2)
#print(len(Bstr2))
s = Bstr0 + Bstr1 + Bstr2
print(len(s))
return height, width, s
user='******',
passwd='admin',
db='stock',
charset='utf8')
cursor = db.cursor()
sql_done_set = "select prevTradeDate from stock_tradecalall a where a.calendarDate >= '%s' and a.exchangeCD = 'XSHG' and isOpen = 1 order by a.prevTradeDate asc" % (
date_seq_start)
cursor.execute(sql_done_set)
done_set = cursor.fetchall()
date_seq = [x[0] for x in done_set]
for i in range(len(date_seq)):
start_dt = (datetime.datetime.strptime(date_seq[i], '%Y-%m-%d') -
datetime.timedelta(days=90)).strftime('%Y-%m-%d')
end_dt = date_seq[i]
dc = DC.data_collect2('002008', start_dt, end_dt)
train = dc.data_train
target = dc.data_target
test_case = np.array([dc.test_case])
#model = ensemble.RandomForestClassifier(n_estimators= 10)
model = svm.SVC(class_weight={1: 10000000})
model.fit(train, target)
ans2 = model.predict(test_case)
print(date_seq[i] + ' ANS : ' + str(ans2[0]))
#target = to_categorical(target, num_classes=None)
#########################################
# MLP 模型
# model = Sequential()
def model_eva(stock,state_dt,para_window,para_dc_window):
# 建立数据库连接，设置tushare token
##################################################
# 建立数据库连接,设置tushare的token,定义一些初始化参数
env=get_env()
db,cursor,pro=env.db,env.cursor,env.pro
##################################################
# print 'into_eva'*10
# 建评估时间序列, para_window参数代表回测窗口长度
model_test_date_start = (datetime.datetime.strptime(state_dt, '%Y-%m-%d') - datetime.timedelta(days=para_window)).strftime(
'%Y%m%d')
model_test_date_end = datetime.datetime.strptime(state_dt,('%Y-%m-%d')).strftime('%Y%m%d')
print "model_test_date_start,model_test_date_end: ",model_test_date_start,model_test_date_end
df = pro.trade_cal(exchange_id='', is_open = 1,start_date=model_test_date_start, end_date=model_test_date_end)
date_temp = list(df.iloc[:,1])
model_test_date_seq = [(datetime.datetime.strptime(x, "%Y%m%d")).strftime('%Y-%m-%d') for x in date_temp]
# 清空评估用的中间表model_ev_mid
sql_truncate_model_test = 'truncate table model_ev_mid'
cursor.execute(sql_truncate_model_test)
db.commit()
# print 'after_commit_'*10
return_flag = 0
# 开始回测，其中para_dc_window参数代表建模时数据预处理所需的时间窗长度
for d in range(len(model_test_date_seq)):
model_test_new_start = (datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d') - datetime.timedelta(days=para_dc_window)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
print "model_test_new_start,model_test_new_end: ",model_test_new_start,model_test_new_end
try:
dc = DC.data_collect(stock, model_test_new_start, model_test_new_end)
if len(set(dc.data_target)) <= 1:
continue
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 将预测结果插入到中间表
# print 'predict'*10
sql_insert = "insert into model_ev_mid(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (model_test_new_end, stock, float(ans2[0]))
# print sql_insert
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
return -1
else:
# 在中间表中刷真实值
for i in range(len(model_test_date_seq)):
sql_select = "select * from stock_all a where a.stock_code = '%s' and a.state_dt >= '%s' order by a.state_dt asc limit 2" % (stock, model_test_date_seq[i])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
if len(done_set2) <= 1:
break
resu = 0
if float(done_set2[1][3]) / float(done_set2[0][3]) > 1.00:
resu = 1
sql_update = "update model_ev_mid w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.stock_code = '%s'" % (resu, model_test_date_seq[i], stock)
cursor.execute(sql_update)
db.commit()
# 计算查全率
sql_resu_recall_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
if recall_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
recall = float(recall_son) / recall_mon
# 计算查准率
sql_resu_acc_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = float(acc_son) / acc_mon
# 计算查准率(负样本)
sql_resu_acc_neg_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg_mon = -1
acc_neg = -1
else:
acc_neg = float(acc_neg_son) / acc_neg_mon
# 计算 F1 分值
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from model_ev_mid a where a.state_dt = '%s'" % (model_test_date_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
# 将评估结果存入结果表model_ev_resu中
sql_final_insert = "insert into model_ev_resu(state_dt,stock_code,acc,recall,f1,acc_neg,bz,predict)values('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (state_dt, stock, acc, recall, f1, acc_neg, 'svm', str(predict))
cursor.execute(sql_final_insert)
db.commit()
db.close()
print(str(state_dt) + ' Precision : ' + str(acc) + ' Recall : ' + str(recall) + ' F1 : ' + str(f1) + ' Acc_Neg : ' + str(acc_neg))
return 1
def filter_main(start_dt, end_dt):
# 建立数据库连接
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='admin',
db='stock',
charset='utf8')
cursor = db.cursor()
#先更新持股天数
sql_update_hold_days = 'update my_stock_pool w set w.hold_days = w.hold_days + 1'
cursor.execute(sql_update_hold_days)
db.commit()
#先卖出
deal = Deal.Deal()
for stock in deal.stock_pool:
init_price = deal.stock_map1[stock]
hold_vol = deal.stock_map2[stock]
hold_days = deal.stock_map3[stock]
sell_price_temp = ts.get_k_data(code=stock, start=start_dt, end=end_dt)
sell_price_list = np.array(sell_price_temp.iloc[0:,
[2]].astype('float32'),
dtype=np.float).ravel()
sell_price = sell_price_list[-1]
if sell_price > init_price * 1.03 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%f','%f','%f','%s','%s','%f','%f','%f','%s','%s','%f')" % (
new_capital, new_money_lock, new_money_rest, 'SELL', stock,
hold_vol, new_profit, new_profit_rate, 'GOODSELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert)
db.commit()
sql_sell_update = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update)
db.commit()
continue
if sell_price < init_price * 0.95 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert2 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%f','%f','%f','%s','%s','%f','%f','%f','%s','%s','%f')" % (
new_capital, new_money_lock, new_money_rest, 'SELL', stock,
hold_vol, new_profit, new_profit_rate, 'BADSELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert2)
db.commit()
sql_sell_update2 = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update2)
db.commit()
sql_ban_insert = "insert into ban_list(stock_code) values ('%s')" % (
stock)
cursor.execute(sql_ban_insert)
db.commit()
continue
if hold_days >= 5 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert3 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%f','%f','%f','%s','%s','%f','%f','%f','%s','%s','%f')" % (
new_capital, new_money_lock, new_money_rest, 'OVERTIME', stock,
hold_vol, new_profit, new_profit_rate, 'OVERTIMESELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert3)
db.commit()
sql_sell_update3 = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update3)
db.commit()
continue
deal_buy = Deal.Deal()
#后买入
if deal_buy.cur_money_rest > 20000:
#Choose.choose_main(start_dt, end_dt)
try:
resu_list = []
score_list = []
#sql_select = "select a.stock_code from q1_cs a where a.calc_date ='%s' order by af/freq desc limit 10" %(end_dt)
sql_select = "select a.stock_code from q1_cs a order by af/freq desc limit 50"
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
for row in done_set2:
if row[0] in deal_buy.ban_list:
continue
dc = DC.data_collect2(row[0], start_dt, end_dt)
if (dc.cnt_good_buy[-1] * dc.cnt_good_sell[-1]) / min(
1, (dc.cnt_bad_sell[-1] + dc.cnt_risk[-1])) < 1:
continue
score_list = []
resu_list = []
train = dc.data_train
target = dc.data_target
model = Sequential()
model.add(Dense(64, activation='linear', input_dim=13))
model.add(Dropout(0.5))
model.add(Dense(64, activation='sigmoid'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='relu'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='logcosh',
optimizer=sgd,
metrics=['accuracy'])
model.fit(train, target, epochs=1000)
score0 = model.evaluate(train, target, batch_size=128)
if float(score0[0]) > 0.5 or str(score0[0]) == 'nan':
continue
for i in range(5):
model.fit(train, target, epochs=5000)
score = model.evaluate(train, target, batch_size=128)
print('SCORE:' + str(score[0]))
test_case = np.array([dc.test_case])
ans2 = model.predict(test_case)
resu_list.append(ans2[0][0])
score_list.append(score)
print('RESU ' + str(i + 1) + ' : ' + str(ans2[0][0]))
dc.refreshDATA(ans2[0][0])
train = dc.data_train
target = dc.data_target
print(score_list)
print(resu_list)
if max(resu_list) > dc.avg[-6] * 1.05 and (
sum(resu_list) / 5) > dc.avg[-6] * 1.02:
vol, rest = divmod(min(deal_buy.cur_money_rest, 30000),
dc.list_close[-6] * 100)
vol = vol * 100
sql_buy_update2 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,state_dt,deal_price)VALUES ('%f', '%f', '%f','%s','%s','%i','%s','%f')" % (
deal_buy.cur_capital -
vol * dc.list_close[-6] * 0.0005,
deal_buy.cur_money_lock + vol * dc.list_close[-6],
deal_buy.cur_money_rest - vol * dc.list_close[-6] *
1.0005, 'buy', row[0], vol, end_dt, dc.list_close[-6])
cursor.execute(sql_buy_update2)
db.commit()
if row[0] in deal_buy.stock_all:
new_buy_price = (deal_buy.stock_map1[row[0]] *
deal_buy.stock_map2[row[0]] +
vol * dc.list_close[-6]) / (
deal_buy.stock_map2[row[0]] + vol)
new_vol = deal_buy.stock_map2[row[0]] + vol
sql_buy_update3 = "update my_stock_pool w set w.buy_price = (select '%f' from dual) where w.stock_code = '%s'" % (
new_buy_price, row[0])
sql_buy_update3b = "update my_stock_pool w set w.hold_vol = (select '%i' from dual) where w.stock_code = '%s'" % (
new_vol, row[0])
sql_buy_update3c = "update my_stock_pool w set w.hold_days = (select '%i' from dual) where w.stock_code = '%s'" % (
1, row[0])
cursor.execute(sql_buy_update3)
cursor.execute(sql_buy_update3b)
cursor.execute(sql_buy_update3c)
db.commit()
else:
sql_buy_update3 = "insert into my_stock_pool(stock_code,buy_price,hold_vol,hold_days) VALUES ('%s','%f','%i','%i')" % (
row[0], dc.list_close[-6], vol, int(1))
cursor.execute(sql_buy_update3)
db.commit()
break
except Exception as excp:
#db.rollback()
print(excp)
db.close()
def model_eva(stock, state_dt, para_window, para_dc_window):
if DBUtils.select_ev_result(state_dt, stock):
print('Already ev:' + stock + ':' + state_dt)
return 0
# 建评估时间序列, para_window参数代表回测窗口长度
ev_start = Utils.date2d(
(Utils.to_date(state_dt) - datetime.timedelta(days=para_window)))
ev_end = state_dt
date_temp = DBUtils.get_stock_calender(ev_start, ev_end)
ev_dt_seq = [(Utils.d2date(x)) for x in date_temp]
# 清空评估用的中间表model_ev_mid
DBUtils.clear_ev_mid()
return_flag = 0
# 开始回测，其中para_dc_window参数代表建模时数据预处理所需的时间窗长度
for d in range(len(ev_dt_seq)):
dc_start_dt = Utils.d2date(
Utils.to_date(ev_dt_seq[d]) -
datetime.timedelta(days=para_dc_window))
dc_end_dt = ev_dt_seq[d]
try:
dc = DC.data_collect(stock, dc_start_dt, dc_end_dt)
if len(set(dc.data_target)) <= 1:
print('WARN: DC target is less than 1 record.')
continue
except Exception as exp:
print("DC Error")
print(exp)
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
aresult = ModelUtils.use_svm(train, target, test_case)
# 将预测结果插入到中间表
DBUtils.insert_predict(dc_end_dt, stock, aresult)
if return_flag == 1:
print('WARN: something maybe wrong... when svm')
acc = recall = acc_neg = f1 = 0
return -1
# 在中间表中刷真实值
for i in range(len(ev_dt_seq)):
r = DBUtils.update_ev_mid_with_real(stock, ev_dt_seq[i])
if r != 0:
print('WARN: break ev mid with real:' + stock)
break
# 计算查全率
recall = DBUtils.count_recall()
# 计算查准率
acc = DBUtils.count_acc()
# 计算查准率(负样本)
acc_neg = DBUtils.count_acc_neg()
# 计算 F1 分值
f1 = Utils.count_F1(acc, recall)
# 将评估结果存入结果表model_ev_resu中
predict = DBUtils.get_predict(ev_dt_seq[-1])
DBUtils.insert_ev_result(state_dt, stock, acc, recall, f1, acc_neg, 'svm',
predict)
print(
str(state_dt) + ' Precision : ' + str(acc) + ' Recall : ' +
str(recall) + ' F1 : ' + str(f1) + ' Acc_Neg : ' + str(acc_neg))
return 1
from sklearn import svm
import DC
if __name__ == '__main__':
stock = '000001.SZ'
dc = DC.data_collect(stock, '20150301', '20190315')
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 输出对2018-03-02的涨跌预测，1表示涨，0表示不涨。
print(ans2[0])
def Status():
flag = False
statusnk = np.zeros([len(files), 6])
statustnk = np.zeros([len(files), 6])
statustk = np.zeros([len(files), 6])
for fID in np.arange(len(files)):
# file 読み込み ------------------------------------------------
print('reading', files[fID])
file = os.path.basename(files[fID])
logFullPath = os.path.join(logsPath, file)
data = open(logFullPath).readlines()
# -------------------------------------------------------------
# 特徴量読み込み -----------------------------------------------
# loading U,B [number of data,10] ※エラーかも
U, B = myData.loadABLV(os.path.join(logsPath, file))
intervalnkY, intervaltnkY, intervaltkY = myData.convV2YearlyData(U)
# -------------------------------------------------------------
# status -----------------------------------------------------
# mean
meannkY = np.mean(intervalnkY)
meantnkY = np.mean(intervaltnkY)
meantkY = np.mean(intervaltkY)
# median
mediannkY = np.median(intervalnkY)
mediantnkY = np.median(intervaltnkY)
mediantkY = np.median(intervaltkY)
# mode
modenkY = stat.mode(intervalnkY)
modetnkY = stat.mode(intervaltnkY)
modetkY = stat.mode(intervaltkY)
# variance
varnkY = np.var(intervalnkY)
vartnkY = np.var(intervaltnkY)
vartkY = np.var(intervaltkY)
# max
maxnkY = np.max(intervalnkY)
maxtnkY = np.max(intervaltnkY)
maxtkY = np.max(intervaltkY)
# min
minnkY = np.min(intervalnkY)
mintnkY = np.min(intervaltnkY)
mintkY = np.min(intervaltkY)
# status in nk
statusnk[fID, 0] = meannkY
statusnk[fID, 1] = mediannkY
statusnk[fID, 2] = modenkY
statusnk[fID, 3] = varnkY
statusnk[fID, 4] = maxnkY
statusnk[fID, 5] = minnkY
# status in tnk
statustnk[fID, 0] = meantnkY
statustnk[fID, 1] = mediantnkY
statustnk[fID, 2] = modetnkY
statustnk[fID, 3] = vartnkY
statustnk[fID, 4] = maxtnkY
statustnk[fID, 5] = mintnkY
# status in tk
statustk[fID, 0] = meantkY
statustk[fID, 1] = mediantkY
statustk[fID, 2] = modetkY
statustk[fID, 3] = vartkY
statustk[fID, 4] = maxtkY
statustk[fID, 5] = mintkY
if not flag:
flag = True
Bs = B
else:
Bs = np.vstack([Bs, B])
# ---------------------------------------------------------------------
# save --------------------------------------------------------------------
np.savetxt(f"{savePath}_{dataPath}_nk.txt", statusnk, fmt="%5f")
np.savetxt(f"{savePath}_{dataPath}_tnk.txt", statustnk, fmt="%5f")
np.savetxt(f"{savePath}_{dataPath}_tk.txt", statustk, fmt="%5f")
np.savetxt(f"{savePath}_{dataPath}_b.txt", Bs, fmt="%6f")
for ind, mode in zip([3], modes[1]):
#path = f"{mode}_{os.path.basename(files[i])}" # nk
path = f"{mode}_{i}" # tnk,tk
myHeatmap.histgram2D(status_tnk[:, ind],
color=f"{colors[cell]}",
sPath=f"{path}")
# -----------------------------------------------------------------------------
if isdeltaU:
logsFullPath = os.path.join(logsPath, "deltaU", fileName)
logfiles = glob.glob(logsFullPath)
flag = False
for file in logfiles:
print(file)
U, _ = myData.loadABLV(file)
yU = myData.convV2YearlyData(U)
if not flag:
yUs = yU
flag = True
else:
yUs = np.vstack([yUs, yU])
myPlot.Histgram(yUs[:, 0], label="yU_nk", color="coral")
myPlot.Histgram(yUs[:, 1], label="yU_tnk", color="forestgreen")
myPlot.Histgram(yUs[:, 2], label="yU_tk", color="royalblue")
def goodpool_init(state_dt):
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='admin',
db='stock',
charset='utf8')
cursor = db.cursor()
sql_select = "select distinct stock_code from stock_all a where a.state_dt = '%s'" % (
str(state_dt))
cursor.execute(sql_select)
done_set = cursor.fetchall()
sql_select2 = "select distinct stock_code from good_pool a"
cursor.execute(sql_select2)
done_set_remove = cursor.fetchall()
ban_list = [x[0] for x in done_set_remove]
stock_pool = [x[0] for x in done_set if x[0] not in ban_list]
model_test_date_start = (datetime.datetime.strptime(state_dt, '%Y-%m-%d') -
datetime.timedelta(days=90)).strftime('%Y-%m-%d')
sql_model_test_date_seq = "select calendarDate from stock_tradecalall a where a.calendarDate >= '%s' and a.calendarDate <= '%s' and a.exchangeCD = 'XSHG' and isOpen = 1 order by a.prevTradeDate asc" % (
model_test_date_start, state_dt)
cursor.execute(sql_model_test_date_seq)
done_set_date_seq = cursor.fetchall()
model_test_date_seq = [x[0] for x in done_set_date_seq]
cnt = 0
for stock in stock_pool:
cnt += 1
sql_truncate_model_test = 'truncate table good_pool_model_test'
cursor.execute(sql_truncate_model_test)
db.commit()
return_flag = 0
for d in range(len(model_test_date_seq)):
model_test_new_start = (
datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d')
- datetime.timedelta(days=365)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
try:
dc = DC.data_collect2(stock, model_test_new_start,
model_test_new_end)
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
if dc.cnt_pos == 0:
return_flag = 1
break
w = 5 * (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
return_flag = 1
break
model = svm.SVC(class_weight={1: w})
model.fit(train, target)
ans2 = model.predict(test_case)
sql_insert = "insert into good_pool_model_test(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (
model_test_new_end, stock, float(ans2[0]))
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
else:
return_flag2 = 0
for i in range(len(model_test_date_seq) - 4):
sql_select = "select * from stock_all a where a.stock_code = '%s' and a.state_dt >= '%s' and a.state_dt <= '%s'" % (
stock, model_test_date_seq[i], model_test_date_seq[i + 4])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
close_list = [x[3] for x in done_set2]
if len(close_list) <= 1:
return_flag2 = 1
break
after_max = max(close_list[1:len(close_list)])
after_min = min(close_list[1:len(close_list)])
resu = 0
if after_max / close_list[0] >= 1.03:
resu = 1
elif after_min / close_list[0] < 0.97:
resu = -1
sql_update = "update good_pool_model_test w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.stock_code = '%s'" % (
resu, model_test_date_seq[i], stock)
cursor.execute(sql_update)
db.commit()
if return_flag2 == 1:
acc = recall = acc_neg = f1 = 0
else:
sql_resu_acc_son = "select count(*) from good_pool_model_test a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from good_pool_model_test a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = acc_son / acc_mon
sql_resu_recall_son = "select count(*) from good_pool_model_test a where a.resu_real is not null and a.resu_predict = a.resu_real"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from good_pool_model_test a where a.resu_real is not null"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
recall = recall_son / recall_mon
sql_resu_acc_neg_son = "select count(*) from good_pool_model_test a where a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = "select count(*) from good_pool_model_test a where a.resu_real is not null and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg_mon = -1
else:
acc_neg = acc_neg_son / acc_neg_mon
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from good_pool_model_test a where a.state_dt = '%s'" % (
model_test_date_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
print(
str(cnt) + ' of total ' + str(len(stock_pool)) + 'Stock : ' +
str(stock) + ' ACC : ' + str(acc) + ' RECALL : ' +
str(recall) + ' ACC_NEG : ' + str(acc_neg) + ' F1 : ' +
str(f1))
sql_final_insert = "insert into good_pool(state_dt,stock_code,acc,recall,f1,acc_neg,bz,predict)values('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (
state_dt, stock, acc, recall, f1, acc_neg, 'svm', str(predict))
cursor.execute(sql_final_insert)
db.commit()
print('ALL FINISHED !!')
db.close()
return 1
from sklearn import svm
import DC
if __name__ == '__main__':
stock = '300672.SZ'
dc = DC.data_collect(stock, '2019-10-01', '2019-10-24')
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 输出对2018-03-02的涨跌预测，1表示涨，0表示不涨。
print(ans2[0])
def filter_main(sql_choose, start_dt, end_dt, slayer):
# 建立数据库连接
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='admin',
db='stock',
charset='utf8')
cursor = db.cursor()
#先更新持股天数
sql_update_hold_days = 'update my_stock_pool w set w.hold_days = w.hold_days + 1'
cursor.execute(sql_update_hold_days)
db.commit()
#先卖出
deal = Deal.Deal()
stock_pool_local = deal.stock_pool
for stock in stock_pool_local:
deal = Deal.Deal()
init_price = deal.stock_map1[stock]
hold_vol = deal.stock_map2[stock]
hold_days = deal.stock_map3[stock]
sql_sell_select = "select * from stock_price_day_list a where a.state_dt = '%s' and a.stock_code = '%s'" % (
end_dt, stock)
cursor.execute(sql_sell_select)
done_set_sell_select = cursor.fetchall()
if len(done_set_sell_select) == 0:
continue
sell_price = float(done_set_sell_select[0][3])
if sell_price > init_price * 1.03 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%.2f','%.2f','%.2f','%s','%s','%.2f','%.2f','%.2f','%s','%s','%.2f')" % (
new_capital, new_money_lock, new_money_rest, 'SELL', stock,
hold_vol, new_profit, new_profit_rate, 'GOODSELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert)
db.commit()
sql_sell_update = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update)
db.commit()
continue
if sell_price < init_price * 0.95 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert2 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%.2f','%.2f','%.2f','%s','%s','%.2f','%.2f','%.2f','%s','%s','%.2f')" % (
new_capital, new_money_lock, new_money_rest, 'SELL', stock,
hold_vol, new_profit, new_profit_rate, 'BADSELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert2)
db.commit()
sql_sell_update2 = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update2)
db.commit()
sql_ban_insert = "insert into ban_list(stock_code) values ('%s')" % (
stock)
cursor.execute(sql_ban_insert)
db.commit()
continue
if hold_days >= 5 and hold_vol > 0:
new_money_lock = deal.cur_money_lock - init_price * hold_vol
new_money_rest = deal.cur_money_rest + sell_price * hold_vol
new_capital = deal.cur_capital + (sell_price -
init_price) * hold_vol
new_profit = (sell_price - init_price) * hold_vol
new_profit_rate = sell_price / init_price
sql_sell_insert3 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,profit,profit_rate,bz,state_dt,deal_price)values('%.2f','%.2f','%.2f','%s','%s','%.2f','%.2f','%.2f','%s','%s','%.2f')" % (
new_capital, new_money_lock, new_money_rest, 'OVERTIME', stock,
hold_vol, new_profit, new_profit_rate, 'OVERTIMESELL', end_dt,
sell_price)
cursor.execute(sql_sell_insert3)
db.commit()
sql_sell_update3 = "delete from my_stock_pool where stock_code = '%s'" % (
stock)
cursor.execute(sql_sell_update3)
db.commit()
continue
deal_buy = Deal.Deal()
#后买入
if deal_buy.cur_money_rest > 20000:
cursor.execute(sql_choose)
done_set2 = cursor.fetchall()
if len(done_set2) > 0:
stock_pool_buy = [x[1] for x in done_set2]
for stock_buy in stock_pool_buy:
sql_stock_check = "SELECT * FROM stock_price_day_list a where stock_code = '%s' and state_dt >= '%s' and state_dt <= '%s' order by state_dt asc" % (
stock_buy, start_dt, end_dt)
cursor.execute(sql_stock_check)
done_set_check = cursor.fetchall()
if len(done_set_check) > 34:
if stock_buy in deal_buy.ban_list:
continue
dc = DC.data_collect2(stock_buy, start_dt, end_dt)
if (dc.cnt_good_buy[-1] * dc.cnt_good_sell[-1]) / min(
1, (dc.cnt_bad_sell[-1] + dc.cnt_risk[-1])) < 1:
continue
score_list = []
resu_list = []
train = dc.data_train
target = dc.data_target
model = Sequential()
model.add(Dense(64, activation='linear', input_dim=14))
model.add(Dropout(0.5))
model.add(Dense(64, activation='sigmoid'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='relu'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='logcosh',
optimizer=sgd,
metrics=['accuracy'])
model.fit(train, target, epochs=1000)
score0 = model.evaluate(train, target, batch_size=128)
if float(score0[0]) > 0.5 or str(score0[0]) == 'nan':
continue
for i in range(5):
model.fit(train, target, epochs=5000)
score = model.evaluate(train, target, batch_size=128)
print('SCORE:' + str(score[0]))
test_case = np.array([dc.test_case])
ans2 = model.predict(test_case)
resu_list.append(ans2[0][0])
score_list.append(score)
print('RESU ' + str(i + 1) + ' : ' +
str(ans2[0][0]))
dc.refreshDATA(ans2[0][0])
train = dc.data_train
target = dc.data_target
print(score_list)
print(resu_list)
if max(resu_list) > dc.avg[-6] * 1.05 and (
sum(resu_list) / 5) > dc.avg[-6] * 1.02:
vol, rest = divmod(min(deal_buy.cur_money_rest, 20000),
dc.close_list[-6] * 100)
vol = vol * 100
sql_buy_update2 = "insert into my_capital(capital,money_lock,money_rest,deal_action,stock_code,stock_vol,state_dt,deal_price,score)VALUES ('%.2f', '%.2f', '%.2f','%s','%s','%i','%s','%.2f','%i')" % (
deal_buy.cur_capital - vol * dc.close_list[-6] *
0.0005, deal_buy.cur_money_lock +
vol * dc.close_list[-6], deal_buy.cur_money_rest -
vol * dc.close_list[-6] * 1.0005, 'buy', stock_buy,
vol, end_dt, dc.close_list[-6], slayer)
cursor.execute(sql_buy_update2)
db.commit()
if stock_buy in deal_buy.stock_all:
new_buy_price = (
deal_buy.stock_map1[stock_buy] *
deal_buy.stock_map2[stock_buy] +
vol * dc.close_list[-6]) / (
deal_buy.stock_map2[stock_buy] + vol)
new_vol = deal_buy.stock_map2[stock_buy] + vol
sql_buy_update3 = "update my_stock_pool w set w.buy_price = (select '%.2f' from dual) where w.stock_code = '%s'" % (
new_buy_price, stock_buy)
sql_buy_update3b = "update my_stock_pool w set w.hold_vol = (select '%i' from dual) where w.stock_code = '%s'" % (
new_vol, stock_buy)
sql_buy_update3c = "update my_stock_pool w set w.hold_days = (select '%i' from dual) where w.stock_code = '%s'" % (
1, stock_buy)
cursor.execute(sql_buy_update3)
cursor.execute(sql_buy_update3b)
cursor.execute(sql_buy_update3c)
db.commit()
else:
sql_buy_update3 = "insert into my_stock_pool(stock_code,buy_price,hold_vol,hold_days) VALUES ('%s','%.2f','%i','%i')" % (
stock_buy, dc.close_list[-6], vol, int(1))
cursor.execute(sql_buy_update3)
db.commit()
break
db.close()
from keras.optimizers import SGD
import DC
from keras.layers import Flatten
import numpy as np
import keras
import datetime
import pymysql
if __name__ == '__main__':
time_temp = datetime.datetime.now() - datetime.timedelta(days=90)
date_seq_start = time_temp.strftime('%Y-%m-%d')
end_dt = (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
# 建立数据库连接,回测时间序列
dc = DC.data_collect2('000725', date_seq_start, end_dt)
score_list = []
resu_list = []
train = dc.data_train
target = dc.data_target
model = Sequential()
model.add(Dense(64, activation='linear', input_dim=14))
model.add(Dropout(0.5))
model.add(Dense(64, activation='sigmoid'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='relu'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='logcosh', optimizer=sgd, metrics=['accuracy'])
for i in range(5):
def future_evaluate(future):
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='admin',
db='future',
charset='utf8')
cursor = db.cursor()
sql_dt_select = "select distinct state_dt from future_all a where a.future_code = '%s' order by a.state_dt asc" % (
future)
cursor.execute(sql_dt_select)
done_set_dt = cursor.fetchall()
date_seq = [x[0] for x in done_set_dt]
start_dt = date_seq[0]
test_seq = date_seq[-31:-1]
sql_truncate_model_test = 'truncate table future_model_test'
cursor.execute(sql_truncate_model_test)
db.commit()
return_flag = 0
for d in range(len(test_seq)):
model_test_new_start = start_dt
model_test_new_end = test_seq[d]
try:
dc = DC.data_collect2(future, model_test_new_start,
model_test_new_end)
except Exception as exp:
print(exp)
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
if dc.cnt_pos == 0:
return_flag = 1
break
w = 5 * (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
return_flag = 1
break
model = svm.SVC(class_weight={1: w})
model.fit(train, target)
ans2 = model.predict(test_case)
sql_insert = "insert into future_model_test(state_dt,future_code,resu_predict)values('%s','%s','%.2f')" % (
model_test_new_end, future, float(ans2[0]))
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
else:
return_flag2 = 0
for i in range(len(test_seq) - 4):
sql_select = "select * from future_all a where a.future_code = '%s' and a.state_dt >= '%s' and a.state_dt <= '%s'" % (
future, test_seq[i], test_seq[i + 4])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
close_list = [x[3] for x in done_set2]
if len(close_list) <= 1:
return_flag2 = 1
break
after_max = max(close_list[1:len(close_list)])
after_min = min(close_list[1:len(close_list)])
resu = 0
if after_max / close_list[0] >= 1.03:
resu = 1
elif after_min / close_list[0] < 0.97:
resu = -1
sql_update = "update future_model_test w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.future_code = '%s'" % (
resu, test_seq[i], future)
cursor.execute(sql_update)
db.commit()
if return_flag2 == 1:
acc = recall = acc_neg = f1 = 0
else:
sql_resu_acc_son = "select count(*) from future_model_test a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from future_model_test a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = acc_son / acc_mon
sql_resu_recall_son = "select count(*) from future_model_test a where a.resu_real is not null and a.resu_predict = a.resu_real"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from future_model_test a where a.resu_real is not null"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
recall = recall_son / recall_mon
sql_resu_acc_neg_son = "select count(*) from future_model_test a where a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = "select count(*) from future_model_test a where a.resu_real is not null and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg_mon = -1
else:
acc_neg = acc_neg_son / acc_neg_mon
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from future_model_test a where a.state_dt = '%s'" % (
test_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
sql_final_insert = "insert into future_good_pool(state_dt,future_code,acc,recall,f1,acc_neg,bz,predict)values('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (
test_seq[-1], future, acc, recall, f1, acc_neg, 'svm', str(predict))
cursor.execute(sql_final_insert)
db.commit()
db.close()
print('Future : ' + str(future) + ' ACC : ' + str(acc) + ' RECALL : ' +
str(recall) + ' ACC_NEG : ' + str(acc_neg) + ' F1 : ' + str(f1))
return 1
cnt = 0
for stock in stock_pool:
cnt += 1
sql_truncate_model_test = 'truncate table good_pool_model_test'
cursor.execute(sql_truncate_model_test)
db.commit()
return_flag = 0
for d in range(len(model_test_date_seq)):
model_test_new_start = (
datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d')
- datetime.timedelta(days=365)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
try:
dc = DC.data_collect2(stock, model_test_new_start,
model_test_new_end)
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
if dc.cnt_pos == 0:
return_flag = 1
break
w = 5 * (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
return_flag = 1
break
model = svm.SVC(class_weight={1: w})
import AC
import DC
import Compress
import cv2
def printBlock(block):
for row in block:
print(row)
img = cv2.imread("../photograph.jpg")
DCT = DCT.DCT()
Quantization = Quantization.Quantization()
AC = AC.AC()
DC = DC.DC()
Compress = Compress.Compress()
Y, U, V = RGB2YUV.rgb2yuv(img, img.shape[1], img.shape[0])
Y = DCT.fill(Y)
blocks = DCT.split(Y)
first = blocks[0]
print('The first block of Y:')
printBlock(first)
print('')
print('The DCT of the block:')
first = DCT.FDCT(first)
printBlock(first)
print('')
print('The Quantization of the DCT:')
first = Quantization.quanY(first)
printBlock(first)
#!/usr/bin/env python
# -*- coding:utf-8 -*-
#@Time : 2019-03-18 14:52
#@Author: FBT
#@File : SVM.py
import DC
from sklearn import svm
if __name__ == '__main__':
stock = '002049.SZ'
dc = DC.data_collect(stock, '2017-03-01', '2018-03-01')
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 输出对2018-03-02的涨跌预测，1表示涨，0表示不涨。
print(ans2[0])
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import SGD
import DC
from keras.layers import Flatten
import numpy as np
import keras
if __name__ == '__main__':
dc = DC.DC('600519')
score_list = []
resu_list = []
train = dc.data_train
target = dc.data_target
model = Sequential()
model.add(Dense(64, activation='linear', input_dim=13))
model.add(Dropout(0.5))
model.add(Dense(64, activation='sigmoid'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='relu'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='logcosh', optimizer=sgd, metrics=['accuracy'])
for i in range(5):
model.fit(train, target, epochs=20000)
score = model.evaluate(train, target, batch_size=128)
print('SCORE:' + str(score[0]))
test_case = np.array([dc.test_case])
ans2 = model.predict(test_case)
resu_list.append(ans2[0][0])
def model_eva(stock, state_dt, para_window, para_dc_window):
# 建立数据库连接，设置tushare token
db = pymysql.connect(host='127.0.0.1',
user='******',
passwd='000109',
db='stock_list',
charset='utf8')
cursor = db.cursor()
ts.set_token('eb13b3bfd2bd07fd9eb40234f19941c73f230e1e98cc212b8cd407c7')
pro = ts.pro_api()
# 建评估时间序列, para_window参数代表回测窗口长度
model_test_date_start = (
datetime.datetime.strptime(state_dt, '%Y-%m-%d') -
datetime.timedelta(days=para_window)).strftime('%Y%m%d')
model_test_date_end = state_dt
df = pro.trade_cal(exchange_id='',
is_open=1,
start_date=model_test_date_start,
end_date=model_test_date_end)
date_temp = list(df.iloc[:, 1])
model_test_date_seq = [
(datetime.datetime.strptime(x, "%Y%m%d")).strftime('%Y-%m-%d')
for x in date_temp
]
# 清空评估用的中间表model_ev_mid
sql_truncate_model_test = 'truncate table model_ev_mid'
cursor.execute(sql_truncate_model_test)
db.commit()
return_flag = 0
# 开始回测，其中para_dc_window参数代表建模时数据预处理所需的时间窗长度
for d in range(len(model_test_date_seq)):
model_test_new_start = (
datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d') -
datetime.timedelta(days=para_dc_window)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
try:
dc = DC.data_collect(stock, model_test_new_start,
model_test_new_end)
if len(set(dc.data_target)) <= 1:
continue
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 将预测结果插入到中间表
sql_insert = "insert into model_ev_mid(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (
model_test_new_end, stock, float(ans2[0]))
cursor.execute(sql_insert)
db.commit()
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
return -1
else:
# 在中间表中刷真实值
for i in range(len(model_test_date_seq)):
sql_select = "select * from stock_all a where a.stock_code = '%s' and a.state_dt >= '%s' order by a.state_dt asc limit 2" % (
stock, model_test_date_seq[i])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
if len(done_set2) <= 1:
break
resu = 0
if float(done_set2[1][3]) / float(done_set2[0][3]) > 1.00:
resu = 1
sql_update = "update model_ev_mid w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.stock_code = '%s'" % (
resu, model_test_date_seq[i], stock)
cursor.execute(sql_update)
db.commit()
# 计算查全率
sql_resu_recall_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
if recall_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
recall = recall_son / recall_mon
# 计算查准率
sql_resu_acc_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = acc_son / acc_mon
# 计算查准率(负样本)
sql_resu_acc_neg_son = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg_mon = -1
acc_neg = -1
else:
acc_neg = acc_neg_son / acc_neg_mon
# 计算 F1 分值
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from model_ev_mid a where a.state_dt = '%s'" % (
model_test_date_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
# 将评估结果存入结果表model_ev_resu中
sql_final_insert = "insert into model_ev_resu(state_dt,stock_code,acc,recall,f1,acc_neg,bz,predict)values('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (
state_dt, stock, acc, recall, f1, acc_neg, 'svm', str(predict))
cursor.execute(sql_final_insert)
db.commit()
db.close()
print('stock:' + stock + str(state_dt) + ' Precision : ' + str(acc) +
' Recall : ' + str(recall) + ' F1 : ' + str(f1) +
' Acc_Neg : ' + str(acc_neg))
return 1
charset='utf8')
cursor = db.cursor()
sql_truncate = "truncate table btc_model_test"
cursor.execute(sql_truncate)
db.commit()
sql_dt_seq = "select distinct state_dt from btc_%smin order by state_dt asc" % (
para_min)
cursor.execute(sql_dt_seq)
done_set_dt = cursor.fetchall()
db.commit()
date_seq = [x[0] for x in done_set_dt]
print_len = len(date_seq) - 1000
for i in range(1900, len(date_seq)):
dc = DC.data_collect2(date_seq[i - 1000], date_seq[i], para_min,
threshold)
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
print(dc.cnt_pos)
if dc.cnt_pos == 0:
return_flag = 1
print('No Positive Samples!')
break
w = (len(target) / dc.cnt_pos)
if len(target) / dc.cnt_pos == 1:
return_flag = 1
print('No Negtive Samples!')
break
#model = svm.SVC(class_weight={1: w})
model = svm.SVC()
def model_eva(stock, state_dt, para_window, para_dc_window):
# 建立数据库连接，设置tushare token
db = pymysql.connect(host='localhost',
user='******',
passwd='your password',
db='your dbname',
charset='utf8mb4')
cursor = db.cursor()
ts.set_token('your token')
pro = ts.pro_api()
# 建评估时间序列, para_window参数代表回测窗口长度
model_test_date_start = (
datetime.datetime.strptime(state_dt, '%Y-%m-%d') -
datetime.timedelta(days=para_window)).strftime('%Y%m%d')
model_test_date_end = datetime.datetime.strptime(
state_dt, '%Y-%m-%d').strftime('%Y%m%d')
# 获取交易日历
df = pro.trade_cal(exchange_id='',
is_open=1,
start_date=model_test_date_start,
end_date=model_test_date_end)
date_temp = list(df.iloc[:, 1])
model_test_date_seq = [
(datetime.datetime.strptime(x, "%Y%m%d")).strftime('%Y-%m-%d')
for x in date_temp
]
# 清空评估用的中间表model_ev_mid
sql_truncate_model_test = 'truncate table model_ev_mid'
cursor.execute(sql_truncate_model_test)
db.commit()
# # 是否清除结果表
# if input('Truncate the model_ev_resu table? Press y/[n]') == 'y':
# sql_truncate_model_resu = 'truncate table model_ev_resu'
# cursor.execute(sql_truncate_model_resu)
# db.commit()
return_flag = 0
# 开始回测，其中para_dc_window参数代表建模时数据预处理所需的时间窗长度
for d in range(len(model_test_date_seq)):
model_test_new_start = (
datetime.datetime.strptime(model_test_date_seq[d], '%Y-%m-%d') -
datetime.timedelta(days=para_dc_window)).strftime('%Y-%m-%d')
model_test_new_end = model_test_date_seq[d]
try:
dc = DC.data_collect(stock, model_test_new_start,
model_test_new_end)
if len(set(dc.data_target)) <= 1:
continue
except Exception as exp:
print("DC Errrrr")
return_flag = 1
break
# 训练模块
method = 'SVM'
train = dc.data_train
target = dc.data_target
test_case = [dc.test_case]
model = svm.SVC() # 建模
model.fit(train, target) # 训练
ans2 = model.predict(test_case) # 预测
# 将预测结果插入到中间表
sql_insert = "insert into model_ev_mid(state_dt,stock_code,resu_predict)values('%s','%s','%.2f')" % (
model_test_new_end, stock, float(ans2[0]))
cursor.execute(sql_insert)
db.commit()
# 如果dc无法获取, 则所有计算值被赋予0，例如股票上市日期不久
if return_flag == 1:
acc = recall = acc_neg = f1 = 0
return -1
else:
# 在中间表中刷真实值
for i in range(len(model_test_date_seq)):
sql_select = "select * from stock_all a where " \
"a.stock_code = '%s' and a.state_dt >= '%s' order by a.state_dt asc limit 2" % (
stock, model_test_date_seq[i])
cursor.execute(sql_select)
done_set2 = cursor.fetchall()
if len(done_set2) <= 1:
break
resu = -1
if float(done_set2[1][3]) / float(done_set2[0][3]) > 1.00:
resu = 1
sql_update = \
"update model_ev_mid w set w.resu_real = '%.2f' where w.state_dt = '%s' and w.stock_code = '%s'" % (
resu, model_test_date_seq[i], stock)
cursor.execute(sql_update)
db.commit()
# 计算查全率
sql_resu_recall_son = "select count(*) from model_ev_mid a where" \
" a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_recall_son)
recall_son = cursor.fetchall()[0][0]
sql_resu_recall_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_real = 1"
cursor.execute(sql_resu_recall_mon)
recall_mon = cursor.fetchall()[0][0]
# 如果统计区间没有实际上涨的情况, 所有指标被赋予0
if recall_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
recall = recall_son / recall_mon
# 计算查准率
sql_resu_acc_son = "select count(*) from model_ev_mid a where" \
" a.resu_real is not null and a.resu_predict = 1 and a.resu_real = 1"
cursor.execute(sql_resu_acc_son)
acc_son = cursor.fetchall()[0][0]
sql_resu_acc_mon = "select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = 1"
cursor.execute(sql_resu_acc_mon)
acc_mon = cursor.fetchall()[0][0]
# 如果没有预测上涨的情况，所有指标被赋予0
if acc_mon == 0:
acc = recall = acc_neg = f1 = 0
else:
acc = acc_son / acc_mon
# 计算查准率(负样本)
sql_resu_acc_neg_son = "select count(*) from model_ev_mid a where" \
" a.resu_real is not null and a.resu_predict = -1 and a.resu_real = -1"
cursor.execute(sql_resu_acc_neg_son)
acc_neg_son = cursor.fetchall()[0][0]
sql_resu_acc_neg_mon = \
"select count(*) from model_ev_mid a where a.resu_real is not null and a.resu_predict = -1"
cursor.execute(sql_resu_acc_neg_mon)
acc_neg_mon = cursor.fetchall()[0][0]
if acc_neg_mon == 0:
acc_neg = -1
else:
acc_neg = acc_neg_son / acc_neg_mon
# 计算 F1 分值
if acc + recall == 0:
acc = recall = acc_neg = f1 = 0
else:
f1 = (2 * acc * recall) / (acc + recall)
sql_predict = "select resu_predict from model_ev_mid a where a.state_dt = '%s'" % (
model_test_date_seq[-1])
cursor.execute(sql_predict)
done_predict = cursor.fetchall()
predict = 0
if len(done_predict) != 0:
predict = int(done_predict[0][0])
# 将评估结果存入结果表model_ev_resu中
sql_final_insert = "insert into model_ev_resu(state_dt,stock_code,acc,recall,f1,acc_neg,bz,predict) values ('%s','%s','%.4f','%.4f','%.4f','%.4f','%s','%s')" % (
state_dt, stock, acc, recall, f1, acc_neg, method, str(predict))
cursor.execute(sql_final_insert)
db.commit()
db.close()
# 打印结果
print(str(state_dt), ' Code:' + str(stock),
' Precision: {:.04f}'.format(acc),
' Recall: {:.04f}'.format(recall), ' F1: {:.04f}'.format(f1),
' Acc_Neg: {:.04f}'.format(acc_neg))
return 1
np.savetxt(os.path.join("research", f"loseB_{mode}.csv"),
worst100B * 1000000,
fmt="%.0f",
delimiter=",")
# -------------------------------------------------------------------------
# interval histgram -----------------------------------------------------------
if isInterval:
# path --------------------------------------------------------------------
logsPath = os.path.join("research", "win", "*txt")
logfiles = glob.glob(logsPath)
# -------------------------------------------------------------------------
for file in logfiles:
V, B = myData.loadABLV(file)
iv_nk, iv_tnk, iv_tk = myData.convV2IntervalData(V)
myPlot.Histgram(iv_nk, label=f"interval_nk", color="coral")
myPlot.Histgram(iv_tnk, label=f"interval_tnk", color="forestgreen")
myPlot.Histgram(iv_tk, label=f"interval_tk", color="royalblue")
if isLoss:
# path --------------------------------------------------------------------
filePath = os.path.join("research", "paramB", "*txt")
files = glob.glob(filePath)
# -------------------------------------------------------------------------
#pdb.set_trace()
# reading paramb files ----------------------------------------------------
atrB = np.loadtxt(files[0])
orB = np.loadtxt(files[2])