def __init__(self, account, password, logger, logdir='./logs/'):
super().__init__(account, password)
self.today = time.strftime('%Y%m%d', time.localtime(time.time()))
if logger is not None:
self.logger = logger
else:
self.logger = log.init_logging(
os.path.join(os.path.expanduser(logdir),
'%s_%s.txt' % (__name__, self.today)), 'info')
self.dbconn = dbpool.MyPymysqlPool(self.logger, 'MysqlDatabaseInfo')
if self.login_status:
self.stock_and_index_list = list(self.getAllStocks().index)
else:
self.stock_and_index_list = []
def make_train_test_data_by_tscode(tscode, seq_len=5):
x = []
y = []
train_x = []
train_y = []
conn = dbpool.MyPymysqlPool(None, 'MysqlDatabaseInfo')
datas = conn.getAll('SELECT `open`, high, low, `close`, `change`, pct_chg, '
'vol, amount, turnover_rate, turnover_rate_f, '
'volume_ratio, pe, pe_ttm, pb, ps, ps_ttm, total_share, float_share, free_share, total_mv, '
'circ_mv FROM t_daily WHERE ts_code="%s"' % tscode)
conn.dispose()
if datas is not None:
for data in datas:
tmp = [data['open'],
data['high'],
data['low'],
data['close'],
data['change'],
data['pct_chg'],
data['vol'],
data['amount'],
data['turnover_rate'],
data['volume_ratio'],
data['total_share'],
data['float_share'],
data['free_share'],
data['total_mv'],
data['circ_mv']]
if tmp.count('nan') or tmp.count('inf') or tmp.count('') or tmp.count('None'):
continue
x.append(tmp)
y.append(data['close'])
for i in range(len(x) - seq_len):
train_x.append(x[i: i + seq_len])
train_y.append(y[i + seq_len])
train_x = np.array(train_x, dtype='float_')
train_y = np.array(train_y, dtype='float_')
ind = int(len(train_x) * 0.8)
return x, y, len(train_x[0][0]), train_x[:ind, :, :], train_y[:ind], train_x[ind:, :, :], train_y[ind:]
def get_newer_five_data(tscode, seq_len=5):
x = []
conn = dbpool.MyPymysqlPool(None, 'MysqlDatabaseInfo')
datas = conn.getAll(
'SELECT * FROM (SELECT trade_date, `open`, high, low, `close`, `change`, pct_chg, vol,'
' amount, turnover_rate, turnover_rate_f, '
'volume_ratio, pe, pe_ttm, pb, ps, ps_ttm,'
' total_share, float_share, free_share, total_mv, '
'circ_mv FROM t_daily WHERE ts_code="%s" order by trade_date desc limit 5)'
' a ORDER BY trade_date' % tscode)
if len(datas) != 5:
return None
for data in datas:
tmp = [
data['open'], data['high'], data['low'], data['close'],
data['change'], data['pct_chg'], data['vol'], data['amount'],
data['turnover_rate'], data['volume_ratio'], data['total_share'],
data['float_share'], data['free_share'], data['total_mv'],
data['circ_mv']
]
x.append(tmp)
x = np.reshape(np.array(x, dtype='float_'), (1, seq_len, len(tmp)))
return x
data['change'], data['pct_chg'], data['vol'], data['amount'],
data['turnover_rate'], data['volume_ratio'], data['total_share'],
data['float_share'], data['free_share'], data['total_mv'],
data['circ_mv']
]
x.append(tmp)
x = np.reshape(np.array(x, dtype='float_'), (1, seq_len, len(tmp)))
return x
if __name__ == '__main__':
stocks_list = []
file_path = 'C:\\Users\\Admin\\Documents\\ZCIT-Projects\\PythonProj\\Quantification\\ml\\stocks_price.txt'
logger = log.init_logging(file_path, 'info')
# 链接数据库
conn = dbpool.MyPymysqlPool(None, 'MysqlDatabaseInfo')
# 获取股票ts代码
for d in conn.getAll('select ts_code from t_stocks'):
stocks_list.append(d['ts_code'])
conn.dispose()
logger.info('tscode\tpred_price\tchange')
for stock in stocks_list:
x = get_newer_five_data(stock, tp.TIMESTEP)
if x is None:
continue
_, price = tp.train(stock, save=False, pred=True, x=x)
logger.info('%s\t%f\t%f' %
(stock, float(price),
(float(price) - x[0][-1][3]) / float(x[0][-1][3])))
if __name__ == '__main__':
date_list = []
stocks_list = []
threads_list = []
cnt = 0
# 初始化日志
if not os.path.exists(os.path.expanduser(logdir)):
os.mkdir(os.path.expanduser(logdir))
logger = log.init_logging(
os.path.join(
os.path.expanduser(logdir), '%s_%s.txt' %
('sync', time.strftime('%Y%m%d', time.localtime(time.time())))),
'info')
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
# 获取股票ts代码
for d in conn.getAll('select ts_code from t_stocks'):
stocks_list.append(d['ts_code'])
for d in conn.getAll(
'select cal_date from t_exchange_trade_cal where cal_date>"20200329" and is_open=1'
):
date_list.append(d['cal_date'])
for d in date_list:
index_sync(logger, conn, stocks_list, d)
today = time.strftime('%Y%m%d', time.localtime(time.time()))
logger.info('-------------%s begin-------------')
logger.info('All %d stocks' % len(stocks_list))
def mymain(args):
stocks_list = []
threads_list = []
# 初始化日志
if not os.path.exists(os.path.expanduser(args.logdir)):
os.mkdir(os.path.expanduser(args.logdir))
logger = log.init_logging(
os.path.join(
os.path.expanduser(args.logdir), '%s_%s.txt' %
(__name__, time.strftime('%Y%m%d', time.localtime(time.time())))),
args.loglevel)
# 初始化tushare
pro = tsapi.init_tushare(logger, tsapi.MYTOKEN)
if pro is None:
print('Init tushare error, exit')
exit(1)
# 链接数据库
conn_tmp = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
# 获取股票ts代码
for d in conn_tmp.getAll('select ts_code from t_stocks'):
stocks_list.append(d['ts_code'])
conn_tmp.dispose()
today = time.strftime('%Y%m%d', time.localtime(time.time()))
if args.startdate == '':
startdate = today
enddate = today
else:
startdate = args.startdate
enddate = args.enddate
# 日行情线程
if args.daily_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_daily = threading.Thread(target=daily,
args=(logger, pro, conn, stocks_list,
startdate, enddate, args.retry))
t_daily.start()
except Exception as e:
logger.error('启动日行情线程失败: %s' % e)
else:
threads_list.append(t_daily)
# 周行情线程
if args.weekly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_weekly = threading.Thread(target=weekly,
args=(logger, pro, conn, stocks_list,
startdate, enddate, args.retry))
t_weekly.start()
except Exception as e:
logger.error('启动周行情线程失败: %s' % e)
else:
threads_list.append(t_weekly)
# 月行情线程
if args.monthly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_monthly = threading.Thread(target=monthly,
args=(logger, pro, conn, stocks_list,
startdate, enddate, args.retry))
t_monthly.start()
except Exception as e:
logger.error('启动月行情线程失败: %s' % e)
else:
threads_list.append(t_monthly)
# 前复权日行情线程
if args.qfq_daily_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_qfq_daily = threading.Thread(target=qfq_daily,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_daily.start()
except Exception as e:
logger.error('启动前复权日行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_daily)
# 前复权周行情线程
if args.qfq_weekly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_qfq_weekly = threading.Thread(target=qfq_weekly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_weekly.start()
except Exception as e:
logger.error('启动前复权周行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_weekly)
# 前复权月行情线程
if args.qfq_monthly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_qfq_monthly = threading.Thread(target=qfq_monthly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_monthly.start()
except Exception as e:
logger.error('启动前复权月行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_monthly)
# 后复权日行情线程
if args.hfq_daily_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_hfq_daily = threading.Thread(target=hfq_daily,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_daily.start()
except Exception as e:
logger.error('启动后复权日行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_daily)
# 后复权周行情线程
if args.hfq_weekly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_hfq_weekly = threading.Thread(target=hfq_weekly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_weekly.start()
except Exception as e:
logger.error('启动后复权周行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_weekly)
# 后复权月行情线程
if args.hfq_monthly_quotation:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_hfq_monthly = threading.Thread(target=hfq_monthly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_monthly.start()
except Exception as e:
logger.error('启动后复权月行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_monthly)
# 每日指标线程
if args.daily_index:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_index = threading.Thread(target=tsindex,
args=(logger, pro, conn, stocks_list,
startdate, enddate, args.retry))
t_index.start()
except Exception as e:
logger.error('启动每日指标线程失败: %s' % e)
else:
threads_list.append(t_index)
# 复权因子线程
if args.adj_factor:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
try:
t_adj = threading.Thread(target=adj_factor,
args=(logger, pro, conn, stocks_list,
args.retry))
t_adj.start()
except Exception as e:
logger.error('启动复权因子线程失败: %s' % e)
else:
threads_list.append(t_adj)
logger.info('Business thread start completed, all %d threads' %
len(threads_list))
def _continueloop(threads):
for t in threads:
if t.is_alive():
return True
return False
now = time.time()
print('\t\t\t\t%s begin' %
time.strftime('%Y%m%d %H%M%S', time.localtime(now)))
while _continueloop(threads_list):
print('\r<%08d>s [\\], see log to more information' %
(time.time() - now),
end='')
time.sleep(0.1)
print('\r<%08d>s [/], see log to more information' %
(time.time() - now),
end='')
time.sleep(0.1)
print('\n\t\t\t\tAll complete, done')
def mymain(args):
stocks_list = []
threads_list = []
today = time.strftime('%Y%m%d', time.localtime(time.time()))
# 初始化日志
if not os.path.exists(os.path.expanduser(args.logdir)):
os.mkdir(os.path.expanduser(args.logdir))
logger = log.init_logging(
os.path.join(os.path.expanduser(args.logdir),
'%s_%s.txt' % (__name__, today)), args.loglevel)
# 初始化tushare
pro = tsapi.init_tushare(logger, tsapi.MYTOKEN)
if pro is None:
print('Init tushare error, exit')
exit(1)
# 链接数据库
conn_sync = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
# 获取股票ts代码
for d in conn_sync.getAll('select ts_code from t_stocks'):
stocks_list.append(d['ts_code'])
logger.info('----%s begin----' %
time.strftime('%Y%m%d %H%M%S', time.localtime(time.time())))
if args.startdate == '':
startdate = today
enddate = today
else:
startdate = args.startdate
enddate = args.enddate
# 日行情线程
if args.daily_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_daily = threading.Thread(target=daily,
args=(logger, pro, conn, startdate,
args.retry))
t_daily.start()
except Exception as e:
logger.error('启动日行情线程失败: %s' % e)
else:
threads_list.append(t_daily)
# 周行情线程
if args.weekly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_weekly = threading.Thread(target=weekly,
args=(logger, pro, conn, startdate,
args.retry))
t_weekly.start()
except Exception as e:
logger.error('启动周行情线程失败: %s' % e)
else:
threads_list.append(t_weekly)
# 月行情线程
if args.monthly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_monthly = threading.Thread(target=monthly,
args=(logger, pro, conn, startdate,
args.retry))
t_monthly.start()
except Exception as e:
logger.error('启动月行情线程失败: %s' % e)
else:
threads_list.append(t_monthly)
# 前复权日行情线程
if args.qfq_daily_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_qfq_daily = threading.Thread(target=qfq_daily,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_daily.start()
except Exception as e:
logger.error('启动前复权日行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_daily)
# 前复权周行情线程
if args.qfq_weekly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_qfq_weekly = threading.Thread(target=qfq_weekly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_weekly.start()
except Exception as e:
logger.error('启动前复权周行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_weekly)
# 前复权月行情线程
if args.qfq_monthly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_qfq_monthly = threading.Thread(target=qfq_monthly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_qfq_monthly.start()
except Exception as e:
logger.error('启动前复权月行情线程失败: %s' % e)
else:
threads_list.append(t_qfq_monthly)
# 后复权日行情线程
if args.hfq_daily_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_hfq_daily = threading.Thread(target=hfq_daily,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_daily.start()
except Exception as e:
logger.error('启动后复权日行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_daily)
# 后复权周行情线程
if args.hfq_weekly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_hfq_weekly = threading.Thread(target=hfq_weekly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_weekly.start()
except Exception as e:
logger.error('启动后复权周行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_weekly)
# 后复权月行情线程
if args.hfq_monthly_quotation:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_hfq_monthly = threading.Thread(target=hfq_monthly,
args=(logger, pro, conn,
stocks_list, startdate,
enddate, args.retry))
t_hfq_monthly.start()
except Exception as e:
logger.error('启动后复权月行情线程失败: %s' % e)
else:
threads_list.append(t_hfq_monthly)
# 每日指标线程
if args.daily_index:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_index = threading.Thread(target=tsindex,
args=(logger, pro, conn, startdate,
args.retry))
t_index.start()
except Exception as e:
logger.error('启动每日指标线程失败: %s' % e)
else:
threads_list.append(t_index)
# 快讯线程
if args.flash_news:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo-news')
t_fnews = threading.Thread(
target=flash_news,
args=(logger, pro, conn, '%s-%s-%s' %
(startdate[:4], startdate[4:6], startdate[6:8]),
'%s-%s-%s' % (enddate[:4], enddate[4:6], enddate[6:8]),
args.retry))
t_fnews.start()
except Exception as e:
logger.error('启动快讯线程失败: %s' % e)
else:
threads_list.append(t_fnews)
# 长篇新闻线程
if args.major_news:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo-news')
t_fnews = threading.Thread(
target=major_news,
args=(logger, pro, conn, '%s-%s-%s' %
(startdate[:4], startdate[4:6], startdate[6:8]),
'%s-%s-%s' % (enddate[:4], enddate[4:6], enddate[6:8]),
args.retry))
t_fnews.start()
except Exception as e:
logger.error('启动长篇新闻线程失败: %s' % e)
else:
threads_list.append(t_fnews)
# 复权因子线程
if args.adj_factor:
try:
conn = dbpool.MyPymysqlPool(logger, 'MysqlDatabaseInfo')
t_adj = threading.Thread(target=adj_factor,
args=(logger, pro, conn, startdate,
args.retry))
t_adj.start()
except Exception as e:
logger.error('启动复权因子线程失败: %s' % e)
else:
threads_list.append(t_adj)
logger.info('Business thread start completed, all %d threads' %
len(threads_list))
def _continueloop(threads):
for t in threads:
if t.is_alive():
return True
return False
now = time.time()
while _continueloop(threads_list):
time.sleep(1)
logger.info('Sync daily quotation and indexs')
index_sync(logger, conn_sync, stocks_list, tradedate=startdate)
conn_sync.dispose()
logger.info('----Complete in [%d] seconds----' % (time.time() - now))