def __init__(self, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.logger = getLogger(__name__)
self.index_objs = dict()
self.stock_objs = dict()
self.updating_date = None
self.combination_objs = dict()
self.cal_client = CCalendar(dbinfo, redis_host)
self.index_info_client = IndexInfo()
self.cvaluation_client = CValuation()
self.reviewer = CReivew(dbinfo, redis_host)
self.comb_info_client = CombinationInfo(dbinfo, redis_host)
self.stock_info_client = CStockInfo(dbinfo, redis_host)
self.rindex_stock_data_client = RIndexStock(dbinfo, redis_host)
self.industry_info_client = IndustryInfo(dbinfo, redis_host)
self.rindustry_info_client = RIndexIndustryInfo(dbinfo, redis_host)
self.animation_client = CAnimation(dbinfo, redis_host)
self.subscriber = Subscriber()
self.quote_handler = StockQuoteHandler()
self.ticker_handler = TickerHandler()
self.connect_client = StockConnect(market_from=ct.SH_MARKET_SYMBOL,
market_to=ct.HK_MARKET_SYMBOL,
dbinfo=dbinfo,
redis_host=redis_host)
self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host)
self.emotion_client = Emotion(dbinfo=dbinfo, redis_host=redis_host)
self.sh_exchange_client = StockExchange(ct.SH_MARKET_SYMBOL)
self.sz_exchange_client = StockExchange(ct.SZ_MARKET_SYMBOL)
def __init__(self, dbinfo = ct.DB_INFO, redis_host = None):
self.dbinfo = dbinfo
self.emotion_table = ct.EMOTION_TABLE
self.redis = create_redis_obj() if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo, iredis = self.redis)
self.rstock_client = RIndexStock(dbinfo, redis_host)
self.logger = getLogger(__name__)
if not self.create(): raise Exception("create emotion table failed")
def __init__(self, index_code, dbinfo = ct.DB_INFO, redis_host = None):
self.dbinfo = dbinfo
self.index_code = index_code
self.index_obj = CIndex(index_code, dbinfo = self.dbinfo, redis_host = redis_host)
self.db_name = self.index_obj.get_dbname(index_code)
self.logger = getLogger(__name__)
self.ris = RIndexStock(dbinfo, redis_host)
self.bull_stock_ratio_table = self.get_table_name()
self.redis = create_redis_obj() if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo, dbname = self.db_name, iredis = self.redis)
if not self.create(): raise Exception("create emotion table failed")
def __init__(self, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.logger = getLogger(__name__)
self.tu_client = get_tushare_client()
self.doc = CDoc()
self.redis = create_redis_obj(
) if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(dbinfo, iredis=self.redis)
self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host)
self.rstock_client = RIndexStock(dbinfo=dbinfo, redis_host=redis_host)
self.sh_market_client = StockExchange(ct.SH_MARKET_SYMBOL)
self.sz_market_client = StockExchange(ct.SZ_MARKET_SYMBOL)
self.emotion_client = Emotion()
def __init__(self):
self.ris = RIndexStock(dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
self.base_color = '#e6daa6'
self.fig = plt.figure(facecolor=self.base_color, figsize=(24, 24))
self.price_ax = plt.subplot2grid((12, 12), (0, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
fig=self.fig)
self.ratio_ax = plt.subplot2grid((12, 12), (6, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
sharex=self.price_ax,
fig=self.fig)
def create_marauder_map_figure(mdate):
df = RIndexStock().get_data(mdate)
TOOLTIPS = [("code", "@code"), ("(pday, profit)", "(@pday, @profit)")]
TOOLS = [BoxZoomTool(), ResetTool(), HoverTool(tooltips=TOOLTIPS)]
p = figure(x_axis_label='时间',
y_axis_label='强度',
tools=TOOLS,
toolbar_location="above",
title="牛熊比例")
if df is None or df.empty: return p
source = ColumnDataSource(df)
color_mapper = LinearColorMapper(palette="Viridis256",
low=df.profit.min(),
high=df.profit.max())
p.circle(x='pday',
y='profit',
color=transform('profit', color_mapper),
size=5,
alpha=0.6,
source=source)
color_bar = ColorBar(color_mapper=color_mapper,
label_standoff=12,
location=(0, 0),
title='强度')
p.add_layout(color_bar, 'right')
return p
def create_stats_figure(mdate):
limit_info = CLimit().get_data(mdate)
stock_info = RIndexStock().get_data(mdate)
stock_info = stock_info[stock_info.volume > 0] #get volume > 0 stock list
stock_info = stock_info.reset_index(drop=True)
limit_up_list = limit_info[(limit_info.pchange > 0)
& (limit_info.prange != 0)].reset_index(
drop=True).code.tolist()
limit_down_list = limit_info[limit_info.pchange < 0].reset_index(
drop=True).code.tolist()
limit_list = limit_up_list + limit_down_list
stock_info = stock_info[~stock_info.code.isin(limit_list)]
changepercent_list = [9, 7, 5, 3, 1, 0, -1, -3, -5, -7, -9]
num_list = list()
name_list = list()
num_list.append(len(limit_up_list))
name_list.append("涨停")
c_length = len(changepercent_list)
for index in range(c_length):
pchange = changepercent_list[index]
if 0 == index:
num_list.append(len(stock_info[stock_info.pchange > pchange]))
name_list.append(">%s" % pchange)
elif c_length - 1 == index:
num_list.append(len(stock_info[stock_info.pchange < pchange]))
name_list.append("<%s" % pchange)
else:
p_max_change = changepercent_list[index - 1]
num_list.append(
len(stock_info[(stock_info.pchange > pchange)
& (stock_info.pchange < p_max_change)]))
name_list.append("%s-%s" % (pchange, p_max_change))
num_list.append(len(limit_down_list))
name_list.append("跌停")
num_list.reverse()
name_list.reverse()
source = ColumnDataSource(
data=dict(names=name_list, values=num_list, colors=Category20_13))
p = figure(x_range=name_list,
y_range=(0, max(num_list) + 100),
title="涨跌幅统计")
p.vbar(x='names', top='values', width=0.9, color='colors', source=source)
p.xgrid.grid_line_color = None
p.add_tools(HoverTool(tooltips=[("涨跌幅", "@names"), ("数量", "@values")]))
return p
def __init__(self):
self.base_color = '#e6daa6'
self.ris = RIndexStock(dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
self.sh_market_client = StockExchange(ct.SH_MARKET_SYMBOL,
dbinfo=ct.OUT_DB_INFO,
redis_host='127.0.0.1')
self.sz_market_client = StockExchange(ct.SZ_MARKET_SYMBOL,
dbinfo=ct.OUT_DB_INFO,
redis_host='127.0.0.1')
self.fig = plt.figure(facecolor=self.base_color, figsize=(24, 24))
self.price_ax = plt.subplot2grid((12, 12), (0, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
fig=self.fig)
self.ratio_ax = plt.subplot2grid((12, 12), (6, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
sharex=self.price_ax,
fig=self.fig)
class OverSell():
def __init__(self):
self.ris = RIndexStock(dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
self.base_color = '#e6daa6'
self.fig = plt.figure(facecolor=self.base_color, figsize=(24, 24))
self.price_ax = plt.subplot2grid((12, 12), (0, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
fig=self.fig)
self.ratio_ax = plt.subplot2grid((12, 12), (6, 0),
rowspan=6,
colspan=12,
facecolor=self.base_color,
sharex=self.price_ax,
fig=self.fig)
def get_data(self, start_date, end_date, index_code):
df = self.ris.get_k_data_in_range(start_date, end_date)
iobj = CIndex(index_code,
dbinfo=ct.OUT_DB_INFO,
redis_host='127.0.0.1')
i_data = iobj.get_k_data_in_range(start_date, end_date)
i_data['time'] = i_data.index.tolist()
i_data = i_data[[
'time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date'
]]
return df, i_data
def is_market_oversell(self, start_date, end_date, index_code):
df, index_data = self.get_data(start_date, end_date, index_code)
info = self.compute_stock_score(df)
def get_oversell_stocks(self, df):
data = df[(np.log(df['close']) - np.log(df['mprice'])) /
np.log(0.9) > 1]
return data.code.tolist()
def compute_stock_score(self, data):
code_list = list()
date_list = list()
rate_list = list()
oversell_ratio = 0
for cdate, df in data.groupby(data.date):
total_num = len(df)
oversell_code_list = self.get_oversell_stocks(df)
oversold_num = len(oversell_code_list)
oversell_ratio = 100 * oversold_num / total_num
date_list.append(cdate)
rate_list.append(oversell_ratio)
code_list.append(oversell_code_list)
info = {'date': date_list, 'rate': rate_list, 'code': code_list}
df = pd.DataFrame(info)
return df
def plot(self, start_date, end_date, index_code):
df, index_data = self.get_data(start_date, end_date, index_code)
date_tickers = index_data.date.tolist()
def _format_date(x, pos=None):
if x < 0 or x > len(date_tickers) - 1: return ''
return date_tickers[int(x)]
info = self.compute_stock_score(df)
candlestick_ohlc(self.price_ax,
index_data.values,
width=1.0,
colorup='r',
colordown='g')
self.ratio_ax.plot(info['date'],
info['rate'],
'r',
label="超跌系数",
linewidth=1)
self.price_ax.xaxis.set_major_locator(mticker.MultipleLocator(20))
self.price_ax.xaxis.set_major_formatter(
mticker.FuncFormatter(_format_date))
plt.show()
def __init__(self):
self.ris = RIndexStock()
self.logger = getLogger(__name__)
class MarauderMap():
def __init__(self):
self.ris = RIndexStock()
self.logger = getLogger(__name__)
def get_data(self, mdate):
return self.ris.get_data(mdate)
def plot(self, cdate, fdir, fname):
df = self.ris.get_data(cdate)
if df.empty: return
fig, ax = plt.subplots()
#get min profit day
min_pday = df.pday.values.min()
max_pday = df.pday.values.max()
#get max profit day
min_profit = df.profit.values.min()
max_profit = df.profit.values.max()
#set axis for map
xmax = max(abs(min_pday), max_pday)
ymax = max(abs(min_profit), max_profit)
ax.set_xlim(-xmax, xmax)
ax.set_ylim(-ymax, ymax)
ax.spines['top'].set_color('none')
ax.spines['right'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data', 0))
for code in df.code.tolist():
pday = df.loc[df.code == code, 'pday']
profit = df.loc[df.code == code, 'profit']
ax.scatter(pday, profit, s=5, alpha=1, linewidths=0.1)
plt.savefig('%s/%s.png' % (fdir, fname), dpi=1000)
def gen_animation(self, end_date, days):
import matplotlib.animation as animation
matplotlib.use('Agg')
start_date = get_day_nday_ago(end_date, num=days, dformat="%Y-%m-%d")
df = self.ris.get_k_data_in_range(start_date, end_date)
fig, ax = plt.subplots()
#get min profit day
min_pday = df.pday.values.min()
max_pday = df.pday.values.max()
#get max profit day
min_profit = df.profit.values.min()
max_profit = df.profit.values.max()
#set axis for map
xmax = max(abs(min_pday), max_pday)
ymax = max(abs(min_profit), max_profit)
groups = df.groupby(df.date)
dates = list(set(df.date.tolist()))
dates.sort()
Writer = animation.writers['ffmpeg']
writer = Writer(fps=2, metadata=dict(artist='biek'), bitrate=-1)
def init():
ax.clear()
ax.set_xlim(-xmax, xmax)
ax.set_ylim(-ymax, ymax)
ax.spines['top'].set_color('none')
ax.spines['right'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data', 0))
def animate(i):
cdate = dates[i]
df = groups.get_group(cdate)
init()
print(cdate, len(df))
bull_stock_num = len(df[df.profit >= 0])
for code in df.code.tolist():
pday = df.loc[df.code == code, 'pday']
profit = df.loc[df.code == code, 'profit']
ax.scatter(pday, profit, color='black', s=1)
ax.set_title("日期：%s 股票总数：%s 牛熊股比:%s" %
(cdate, len(df), 100 * bull_stock_num / len(df)),
fontproperties=get_chinese_font())
ani = animation.FuncAnimation(fig,
animate,
frames=len(dates),
init_func=init,
interval=1000,
repeat=False)
sfile = '/code/panimation.mp4'
ani.save(sfile, writer)
ax.set_title('Marauder Map for date')
ax.grid(True)
plt.close(fig)
class MarauderMap():
def __init__(self, code_list):
self.codes = code_list
self.ris = RIndexStock()
self.logger = getLogger(__name__)
def plot(self, cdate, fdir, fname):
df = self.ris.get_data(cdate)
if df.empty: return
fig, ax = plt.subplots()
#get min profit day
min_pday = df.pday.values.min()
max_pday = df.pday.values.max()
#get max profit day
min_profit = df.profit.values.min()
max_profit = df.profit.values.max()
#set axis for map
xmax = max(abs(min_pday), max_pday)
ymax = max(abs(min_profit), max_profit)
ax.set_xlim(-xmax, xmax)
ax.set_ylim(-ymax, ymax)
ax.spines['top'].set_color('none')
ax.spines['right'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data', 0))
for code in self.codes:
pday = df.loc[df.code == code, 'pday']
profit = df.loc[df.code == code, 'profit']
ax.scatter(pday, profit, s=5, alpha=1, linewidths=0.1)
plt.savefig('%s/%s.png' % (fdir, fname), dpi=1000)
def gen_animation(self, cdate):
df = self.get_data(cdate)
fig, ax = plt.subplots()
Writer = animation.writers['ffmpeg']
writer = Writer(fps=30, metadata=dict(artist='biek'), bitrate=1800)
#get min profit day
min_pday = df.pday.values.min()
max_pday = df.pday.values.max()
#get max profit day
min_profit = df.profit.values.min()
max_profit = df.profit.values.max()
#set axis for map
xmax = max(abs(min_pday), max_pday)
ymax = max(abs(min_profit), max_profit)
groups = list(df.groupby(df.time))
def init():
ax.clear()
ax.set_xlim(-xmax, xmax)
ax.set_ylim(-ymax, ymax)
ax.spines['top'].set_color('none')
ax.spines['right'].set_color('none')
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data', 0))
def animate(n):
val = groups[n][1].values.tolist()[0]
for code in self.codes:
pday = val[1]
profit = val[2]
ax.scatter(pday, profit, s=5, alpha=1, linewidths=0.1)
ani = animation.FuncAnimation(fig,
animate,
frames=300,
init_func=init,
interval=1,
repeat=False)
sfile = '/code/animation.mp4'
ani.save(sfile, writer, fps=60, dpi=100)
ax.set_title('Marauder Map for date')
ax.grid(True)
plt.close(fig)
class DataManager:
def __init__(self, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.logger = getLogger(__name__)
self.index_objs = dict()
self.stock_objs = dict()
self.updating_date = None
self.combination_objs = dict()
self.cal_client = CCalendar(dbinfo, redis_host)
self.index_info_client = IndexInfo()
self.cvaluation_client = CValuation()
self.reviewer = CReivew(dbinfo, redis_host)
self.comb_info_client = CombinationInfo(dbinfo, redis_host)
self.stock_info_client = CStockInfo(dbinfo, redis_host)
self.rindex_stock_data_client = RIndexStock(dbinfo, redis_host)
self.industry_info_client = IndustryInfo(dbinfo, redis_host)
self.rindustry_info_client = RIndexIndustryInfo(dbinfo, redis_host)
self.animation_client = CAnimation(dbinfo, redis_host)
self.subscriber = Subscriber()
self.quote_handler = StockQuoteHandler()
self.ticker_handler = TickerHandler()
self.connect_client = StockConnect(market_from=ct.SH_MARKET_SYMBOL,
market_to=ct.HK_MARKET_SYMBOL,
dbinfo=dbinfo,
redis_host=redis_host)
self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host)
self.emotion_client = Emotion(dbinfo=dbinfo, redis_host=redis_host)
self.sh_exchange_client = StockExchange(ct.SH_MARKET_SYMBOL)
self.sz_exchange_client = StockExchange(ct.SZ_MARKET_SYMBOL)
def is_collecting_time(self):
now_time = datetime.now()
_date = now_time.strftime('%Y-%m-%d')
y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3]
aft_open_hour, aft_open_minute, aft_open_second = (17, 10, 00)
aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute,
aft_open_second)
aft_close_hour, aft_close_minute, aft_close_second = (23, 59, 59)
aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute,
aft_close_second)
#self.logger.info("collecting now time. open_time:%s < now_time:%s < close_time:%s" % (aft_open_time, now_time, aft_close_time))
return aft_open_time < now_time < aft_close_time
def is_morning_time(self, now_time=datetime.now()):
_date = now_time.strftime('%Y-%m-%d')
y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3]
mor_open_hour, mor_open_minute, mor_open_second = (0, 0, 0)
mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute,
mor_open_second)
mor_close_hour, mor_close_minute, mor_close_second = (6, 30, 0)
mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute,
mor_close_second)
return mor_open_time < now_time < mor_close_time
def collect_combination_runtime_data(self):
def _combination_run(code_id):
self.combination_objs[code_id].run()
return (code_id, True)
todo_iplist = list(self.combination_objs.keys())
return concurrent_run(_combination_run, todo_iplist, num=10)
def collect_stock_runtime_data(self):
if self.ticker_handler.empty(): return
datas = self.ticker_handler.getQueue()
while not datas.empty():
df = datas.get()
df = df.set_index('time')
df.index = pd.to_datetime(df.index)
for code_str in set(df.code):
code_id = code_str.split('.')[1]
self.stock_objs[code_id].run(df.loc[df.code == code_str])
def init_real_stock_info(self):
concerned_list = self.comb_info_client.get_concerned_list()
prefix_concerned_list = [add_prifix(code) for code in concerned_list]
ret = self.subscriber.subscribe(prefix_concerned_list, SubType.TICKER,
self.ticker_handler)
if 0 == ret:
for code in concerned_list:
if code not in self.stock_objs:
self.stock_objs[code] = CStock(code,
self.dbinfo,
should_create_influxdb=True,
should_create_mysqldb=True)
return ret
def init_index_info(self):
index_list = ct.INDEX_DICT.keys()
prefix_index_list = [add_index_prefix(code) for code in index_list]
ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE,
self.quote_handler)
if 0 != ret:
self.logger.error("subscribe for index list failed")
return ret
for code in index_list:
if code not in self.index_objs:
self.index_objs[code] = CIndex(code,
should_create_influxdb=True,
should_create_mysqldb=True)
return 0
def collect_index_runtime_data(self):
if self.quote_handler.empty(): return
datas = self.quote_handler.getQueue()
while not datas.empty():
df = datas.get()
df['time'] = df.data_date + ' ' + df.data_time
df = df.drop(['data_date', 'data_time'], axis=1)
df = df.set_index('time')
df.index = pd.to_datetime(df.index)
for code_str in set(df.code):
code_id = code_str.split('.')[1]
self.index_objs[code_id].run(df.loc[df.code == code_str])
def run(self, sleep_time):
while True:
try:
self.logger.debug("enter run")
if self.cal_client.is_trading_day():
if is_trading_time():
t_sleep_time = 1
if not self.subscriber.status():
self.subscriber.start()
if 0 == self.init_index_info(
) and 0 == self.init_real_stock_info():
self.init_combination_info()
else:
self.logger.debug("enter stop subscriber")
self.subscriber.stop()
else:
self.collect_stock_runtime_data()
self.collect_combination_runtime_data()
self.collect_index_runtime_data()
self.animation_client.collect()
else:
t_sleep_time = sleep_time
if self.subscriber.status():
self.subscriber.stop()
else:
t_sleep_time = sleep_time
except Exception as e:
#traceback.print_exc()
self.logger.error(e)
gevent.sleep(t_sleep_time)
def set_update_info(self,
step_length,
exec_date,
cdate=None,
filename=ct.STEPFILE):
step_info = dict()
if cdate is None: cdate = 'none'
step_info[cdate] = dict()
step_info[cdate]['step'] = step_length
step_info[cdate]['date'] = exec_date
with open(filename, 'w') as f:
json.dump(step_info, f)
self.logger.info("finish step :%s" % step_length)
def get_update_info(self,
cdate=None,
exec_date=None,
filename=ct.STEPFILE):
if cdate is None: cdate = 'none'
if not os.path.exists(filename): return (0, exec_date)
with open(filename, 'r') as f:
step_info = json.load(f)
if cdate not in step_info: return (0, exec_date)
return (step_info[cdate]['step'], step_info[cdate]['date'])
def bootstrap(self,
cdate=None,
exec_date=datetime.now().strftime('%Y-%m-%d'),
ndays=3):
finished_step, exec_date = self.get_update_info(cdate, exec_date)
self.logger.info("enter updating.%s" % finished_step)
if finished_step < 1:
if not self.cal_client.init():
self.logger.error("cal client init failed")
return False
self.set_update_info(1, exec_date, cdate)
if finished_step < 2:
if not self.index_info_client.update():
self.logger.error("index info init failed")
return False
self.set_update_info(2, exec_date, cdate)
if finished_step < 3:
if not self.stock_info_client.update():
self.logger.error("stock info init failed")
return False
self.set_update_info(3, exec_date, cdate)
if finished_step < 4:
if not self.comb_info_client.update():
self.logger.error("comb info init failed")
return False
self.set_update_info(4, exec_date, cdate)
if finished_step < 5:
if not self.industry_info_client.update():
self.logger.error("industry info init failed")
return False
self.set_update_info(5, exec_date, cdate)
if finished_step < 6:
if not self.init_tdx_index_info(cdate):
self.logger.error("init tdx index info failed")
return False
self.set_update_info(6, exec_date, cdate)
if finished_step < 7:
if not self.sh_exchange_client.update(exec_date, num=ndays):
self.logger.error("sh exchange update failed")
return False
self.set_update_info(7, exec_date, cdate)
if finished_step < 8:
if not self.sz_exchange_client.update(exec_date, num=ndays):
self.logger.error("sz exchange update failed")
return False
self.set_update_info(8, exec_date, cdate)
if finished_step < 9:
if not self.init_index_components_info(exec_date):
self.logger.error("init index components info failed")
return False
self.set_update_info(9, exec_date, cdate)
if finished_step < 10:
if not self.init_industry_info(cdate):
self.logger.error("init industry info failed")
return False
self.set_update_info(10, exec_date, cdate)
if finished_step < 11:
if not self.rindustry_info_client.update(exec_date, num=ndays):
self.logger.error("init %s rindustry info failed" % exec_date)
return False
self.set_update_info(11, exec_date, cdate)
if finished_step < 12:
if not self.init_yesterday_hk_info(exec_date, num=ndays):
self.logger.error("init yesterday hk info failed")
return False
self.set_update_info(12, exec_date, cdate)
if finished_step < 13:
if not self.margin_client.update(exec_date, num=ndays):
self.logger.error("init yesterday margin failed")
return False
self.set_update_info(13, exec_date, cdate)
if finished_step < 14:
if not self.init_stock_info(cdate):
self.logger.error("init stock info set failed")
return False
self.set_update_info(14, exec_date, cdate)
if finished_step < 15:
if not self.init_base_float_profit():
self.logger.error("init base float profit for all stock")
return False
self.set_update_info(15, exec_date, cdate)
if finished_step < 16:
if not self.init_valuation_info(cdate):
self.logger.error("init stock valuation info failed")
return False
self.set_update_info(16, exec_date, cdate)
if finished_step < 17:
if not self.init_rvaluation_info(cdate):
self.logger.error("init r stock valuation info failed")
return False
self.set_update_info(17, exec_date, cdate)
if finished_step < 18:
if not self.init_rindex_valuation_info(cdate):
self.logger.error("init r index valuation info failed")
return False
self.set_update_info(18, exec_date, cdate)
if finished_step < 19:
if not self.rindex_stock_data_client.update(exec_date, num=ndays):
self.logger.error("rstock data set failed")
return False
self.set_update_info(19, exec_date, cdate)
if finished_step < 20:
if not self.set_bull_stock_ratio(exec_date, num=ndays):
self.logger.error("bull ratio set failed")
return False
self.set_update_info(20, exec_date, cdate)
self.logger.info("updating succeed")
return True
def clear_network_env(self):
kill_process("google-chrome")
kill_process("renderer")
kill_process("Xvfb")
kill_process("zygote")
kill_process("defunct")
kill_process("show-component-extension-options")
def update(self, sleep_time):
succeed = False
while True:
self.logger.debug("enter daily update process. %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
try:
if self.cal_client.is_trading_day():
#self.logger.info("is trading day. %s, succeed:%s" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), succeed))
if self.is_collecting_time():
self.logger.debug(
"enter collecting time. %s, succeed:%s" %
(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
succeed))
if not succeed:
self.clear_network_env()
mdate = datetime.now().strftime('%Y-%m-%d')
ndate = get_latest_data_date()
if ndate is not None:
if ndate >= transfer_date_string_to_int(mdate):
if self.updating_date is None:
self.updating_date = mdate
succeed = self.bootstrap(
cdate=self.updating_date,
exec_date=self.updating_date)
if succeed: self.updating_date = None
else:
self.logger.debug("%s is older for %s" %
(ndate, mdate))
else:
succeed = False
gevent.sleep(sleep_time)
except Exception as e:
time.sleep(1)
self.logger.error(e)
def init_combination_info(self):
trading_info = self.comb_info_client.get()
for _, code_id in trading_info['code'].iteritems():
if str(code_id) not in self.combination_objs:
self.combination_objs[str(code_id)] = Combination(
code_id, self.dbinfo)
def init_base_float_profit(self):
def _set_base_float_profit(code_id):
if CStock(code_id).set_base_floating_profit():
self.logger.info("%s set base float profit success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set base float profit failed" % code_id)
return (code_id, False)
df = self.stock_info_client.get()
if df.empty: return False
failed_list = df.code.tolist()
return process_concurrent_run(_set_base_float_profit,
failed_list,
num=8)
def init_rindex_valuation_info(self, cdate):
for code in ct.INDEX_DICT:
if not self.cvaluation_client.set_index_valuation(code, cdate):
self.logger.error(
"{} set {} data for rvaluation failed".format(code, mdate))
return False
return True
def init_rvaluation_info(self, cdate=None):
def cget(mdate, code):
return code, CStock(code).get_val_data(mdate)
df = self.stock_info_client.get()
code_list = df.code.tolist()
try:
obj_pool = Pool(5000)
all_df = pd.DataFrame()
cfunc = partial(cget, cdate)
for code_data in obj_pool.imap_unordered(cfunc, code_list):
if code_data[1] is not None and not code_data[1].empty:
tem_df = code_data[1]
tem_df['code'] = code_data[0]
all_df = all_df.append(tem_df)
obj_pool.join(timeout=5)
obj_pool.kill()
all_df = all_df.reset_index(drop=True)
file_name = "{}.csv".format(cdate)
file_path = Path(ct.RVALUATION_DIR) / file_name
all_df.to_csv(file_path,
index=False,
header=True,
mode='w',
encoding='utf8')
return True
except Exception as e:
self.logger.error(e)
return False
def init_valuation_info(self, cdate=None):
df = self.stock_info_client.get()
code_list = df['code'].tolist()
time2market_list = df['timeToMarket'].tolist()
code2timedict = dict(zip(code_list, time2market_list))
cfun = partial(self.cvaluation_client.set_stock_valuation,
code2timedict, cdate)
return process_concurrent_run(cfun,
code_list,
num=15,
black_list=list())
def init_stock_info(self, cdate=None):
def _set_stock_info(mdate, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, mdate):
self.logger.info("%s set k data success for date:%s",
code_id, mdate)
return (code_id, True)
else:
self.logger.error("%s set k data failed for date:%s",
code_id, mdate)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data for date %s exception:%s",
code_id, mdate, e)
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
if df.empty: return False
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=8)
else:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
succeed = True
if not process_concurrent_run(cfunc, failed_list, num=8):
succeed = False
return succeed
#start_date = get_day_nday_ago(cdate, num = 4, dformat = "%Y-%m-%d")
#for mdate in get_dates_array(start_date, cdate, asending = True):
# if self.cal_client.is_trading_day(mdate):
# self.logger.info("start recording stock info: %s", mdate)
# cfunc = partial(_set_stock_info, mdate, bonus_info, index_info)
# if not process_concurrent_run(cfunc, failed_list, num = 500):
# self.logger.error("compute stock info for %s failed", mdate)
# return False
#return True
def init_industry_info(self, cdate, num=1):
def _set_industry_info(cdate, code_id):
return (code_id, CIndex(code_id).set_k_data(cdate))
df = self.industry_info_client.get()
if cdate is None:
cfunc = partial(_set_industry_info, cdate)
return concurrent_run(cfunc, df.code.tolist(), num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_industry_info, mdate)
if not concurrent_run(cfunc, df.code.tolist(), num=5):
succeed = False
return succeed
def init_yesterday_hk_info(self, cdate, num):
succeed = True
for data in ((ct.SH_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL),
(ct.SZ_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL)):
if not self.connect_client.set_market(data[0], data[1]):
self.logger.error("connect_client for %s failed" % data)
succeed = False
continue
if not self.connect_client.update(cdate, num=num):
succeed = False
self.connect_client.close()
self.connect_client.quit()
return succeed
def get_concerned_index_codes(self):
index_codes = list(ct.INDEX_DICT.keys())
#添加MSCI板块
index_codes.append('880883')
return index_codes
def init_index_components_info(self, cdate=None):
if cdate is None: cdate = datetime.now().strftime('%Y-%m-%d')
def _set_index_info(code_id):
if code_id in self.index_objs:
_obj = self.index_objs[code_id]
else:
_obj = CIndex(code_id) if code_id in list(
ct.INDEX_DICT.keys()) else TdxFgIndex(code_id)
return (code_id, _obj.set_components_data(cdate))
index_codes = self.get_concerned_index_codes()
return concurrent_run(_set_index_info, index_codes, num=10)
def set_bull_stock_ratio(self, cdate, num=10):
def _set_bull_stock_ratio(code_id):
return (code_id, BullStockRatio(code_id).update(cdate, num))
index_codes = self.get_concerned_index_codes()
return concurrent_run(_set_bull_stock_ratio, index_codes)
def init_tdx_index_info(self, cdate=None, num=1):
def _set_index_info(cdate, code_id):
try:
if code_id in self.index_objs:
_obj = self.index_objs[code_id]
else:
_obj = CIndex(code_id) if code_id in list(
ct.TDX_INDEX_DICT.keys()) else TdxFgIndex(code_id)
return (code_id, _obj.set_k_data(cdate))
except Exception as e:
self.logger.error(e)
return (code_id, False)
#index_code_list = self.get_concerned_index_codes()
index_code_list = list(ct.TDX_INDEX_DICT.keys())
if cdate is None:
cfunc = partial(_set_index_info, cdate)
return concurrent_run(cfunc, index_code_list, num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_index_info, mdate)
if not concurrent_run(cfunc, index_code_list, num=5):
succeed = False
return succeed
class Emotion:
def __init__(self, dbinfo = ct.DB_INFO, redis_host = None):
self.dbinfo = dbinfo
self.emotion_table = ct.EMOTION_TABLE
self.redis = create_redis_obj() if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo, iredis = self.redis)
self.rstock_client = RIndexStock(dbinfo, redis_host)
self.logger = getLogger(__name__)
if not self.create(): raise Exception("create emotion table failed")
def create(self):
if self.emotion_table not in self.mysql_client.get_all_tables():
sql = 'create table if not exists %s(date varchar(10) not null, score float, PRIMARY KEY (date))' % self.emotion_table
if not self.mysql_client.create(sql, self.emotion_table): return False
return True
def get_score(self, cdate = None):
if cdate is None:
sql = "select * from %s" % self.emotion_table
else:
sql = "select * from %s where date=\"%s\"" %(self.emotion_table, cdate)
return self.mysql_client.get(sql)
def get_stock_data(self, cdate):
df_byte = self.redis.get(ct.TODAY_ALL_STOCK)
if df_byte is None: return None
df = _pickle.loads(df_byte)
return df.loc[df.date == date]
def update(self, end_date = None, num = 3):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d")
succeed = True
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis = self.redis):
if not self.set_score(mdate):
succeed = False
self.logger.info("set score for %s set failed" % mdate)
return succeed
def set_score(self, cdate = datetime.now().strftime('%Y-%m-%d')):
stock_info = self.rstock_client.get_data(cdate)
limit_info = CLimit(self.dbinfo).get_data(cdate)
if stock_info.empty or limit_info.empty:
self.logger.error("info is empty failed")
return False
limit_up_list = limit_info[(limit_info.pchange > 0) & (limit_info.prange != 0)].reset_index(drop = True).code.tolist()
limit_down_list = limit_info[limit_info.pchange < 0].reset_index(drop = True).code.tolist()
limit_up_list.extend(limit_down_list)
total = 0
for _index, pchange in stock_info.pchange.iteritems():
code = stock_info.loc[_index, 'code']
if code in limit_up_list:
total += 2 * pchange
else:
total += pchange
aver = total / len(stock_info)
data = {'date':[cdate], 'score':[aver]}
df = pd.DataFrame.from_dict(data)
return self.mysql_client.set(df, self.emotion_table)
sh_df = get_market_data(ct.SH_MARKET_SYMBOL, start_date, end_date)
sz_df = get_market_data(ct.SZ_MARKET_SYMBOL, start_date, end_date)
date_list = list(set(sh_df.date.tolist()).intersection(set(sz_df.date.tolist())))
sh_df = sh_df[sh_df.date.isin(date_list)]
sz_df = sz_df[sz_df.date.isin(date_list)]
#rzrq info
sh_rzrq_df = get_rzrq_info(ct.SH_MARKET_SYMBOL, start_date, end_date)
sz_rzrq_df = get_rzrq_info(ct.SZ_MARKET_SYMBOL, start_date, end_date)
date_list = list(set(sh_rzrq_df.date.tolist()).intersection(set(sz_rzrq_df.date.tolist())))
sh_rzrq_df = sh_rzrq_df[sh_rzrq_df.date.isin(date_list)]
sz_rzrq_df = sz_rzrq_df[sz_rzrq_df.date.isin(date_list)]
#average price info
av_df = get_index_df('880003', start_date, end_date)
#limit up and down info
limit_info = CLimit().get_data(cdate)
stock_info = RIndexStock().get_data(cdate)
stock_info = stock_info[stock_info.volume > 0] #get volume > 0 stock list
stock_info = stock_info.reset_index(drop = True)
#index info
index_info = get_index_data(end_date)
#industry analysis
industry_info = get_industry_data(cdate)
#all stock info
all_stock_info = RIndexStock().get_k_data_in_range(start_date, end_date)
stm = StrongerThanMarketSelecter()
stm_code_list = stm.choose(all_stock_info, av_df)
amus = AntiMarketUpSelecter()
amus_code_list = amus.choose(stock_info)
import os
import sys
from os.path import abspath, dirname
sys.path.insert(0, dirname(dirname(dirname(abspath(__file__)))))
import traceback
import const as ct
import pandas as pd
from rstock import RIndexStock
from cstock_info import CStockInfo
from industry_info import IndustryInfo
if __name__ == '__main__':
try:
mdate = '2019-08-02'
cobj = CStockInfo()
robj = RIndexStock()
iobj = IndustryInfo()
black_list = list(ct.BLACK_DICT.keys())
bdf = cobj.get()
stock_info = robj.get_data(mdate)
idf = iobj.get_csi_industry_data(mdate)
df = pd.merge(bdf, idf, how='left', on=['code'])
df = pd.merge(stock_info, df, how='inner', on=['code'])
df = df[~df.code.isin(black_list)]
df = df[(df.profit > 1) & (df.profit < 3) & (df.pday > 30) &
(df.timeToMarket < 20150101)]
df = df.reset_index(drop=True)
#df = df[['code', 'name', 'industry', 'profit', 'pday', 'pind_name', 'sind_name', 'tind_name', 'find_name']]
df = df[['code', 'name', 'profit', 'pday', 'find_name']]
for name, contains in df.groupby('find_name'):
def __init__(self, code_list):
self.codes = code_list
self.ris = RIndexStock()
self.logger = getLogger(__name__)
def __init__(self, dinfo, rhost):
self.dbinfo = dinfo
self.redis_host = rhost
self.base_color = '#e6daa6'
self.ris = RIndexStock(dbinfo=self.dbinfo, redis_host=self.redis_host)
return 1
elif code in black_set:
return -1
else:
return 0
#if __name__ == '__main__':
# mval_client = MValuation()
# mval_client.update_index(end_date = '2019-08-13')
if __name__ == '__main__':
try:
mdate = 20190815
mval_client = MValuation()
rindex_client = RIndexStock()
df = rindex_client.get_data(transfer_int_to_date_string(mdate))
df['mv'] = df['totals'] * df['close']
df['mv'] = df['mv'] / 100000000
df = df[df.mv > 100]
df = df[(df.profit > 1) & (df.profit < 6.5)]
#黑名单
black_set = set(ct.BLACK_DICT.keys())
white_set = set(ct.WHITE_DICT.keys())
if len(black_set.intersection(white_set)) > 0:
raise Exception("black and white has intersection.")
df['history'] = df.apply(
lambda row: get_hist_val(black_set, white_set, row['code']),
axis=1)
df = df[df['history'] > -1]
#basic_info
def get_stock_data(start_date, end_date):
ris = RIndexStock(ct.OUT_DB_INFO, redis_host='127.0.0.1')
return ris.get_k_data_in_range(start_date, end_date)
class CReivew:
def __init__(self, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.logger = getLogger(__name__)
self.tu_client = get_tushare_client()
self.doc = CDoc()
self.redis = create_redis_obj(
) if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(dbinfo, iredis=self.redis)
self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host)
self.rstock_client = RIndexStock(dbinfo=dbinfo, redis_host=redis_host)
self.sh_market_client = StockExchange(ct.SH_MARKET_SYMBOL)
self.sz_market_client = StockExchange(ct.SZ_MARKET_SYMBOL)
self.emotion_client = Emotion()
def get_industry_data(self, cdate):
ri = RIndexIndustryInfo()
df = ri.get_k_data(cdate)
if df.empty: return df
df = df.reset_index(drop=True)
df = df.sort_values(by='amount', ascending=False)
df['money_change'] = (df['amount'] - df['preamount']) / 1e8
industry_info = IndustryInfo.get()
df = pd.merge(df, industry_info, how='left', on=['code'])
return df
def get_index_data(self, cdate):
df = pd.DataFrame()
for code, name in ct.TDX_INDEX_DICT.items():
data = CIndex(code).get_k_data(cdate)
data['name'] = name
data['code'] = code
df = df.append(data)
df = df.reset_index(drop=True)
return df
def get_market_data(self, market, start_date, end_date):
if market == ct.SH_MARKET_SYMBOL:
df = self.sh_market_client.get_k_data_in_range(
start_date, end_date)
df = df.loc[df.name == '上海市场']
else:
df = self.sz_market_client.get_k_data_in_range(
start_date, end_date)
df = df.loc[df.name == '深圳市场']
df = df.round(2)
df = df.drop_duplicates()
df = df.reset_index(drop=True)
df = df.sort_values(by='date', ascending=True)
df.negotiable_value = (df.negotiable_value / 2).astype(int)
return df
def get_rzrq_info(self, market, start_date, end_date):
df = self.margin_client.get_k_data_in_range(start_date, end_date)
if market == ct.SH_MARKET_SYMBOL:
df = df.loc[df.code == 'SSE']
df['code'] = '上海市场'
else:
df = df.loc[df.code == 'SZSE']
df['code'] = '深圳市场'
df = df.round(2)
df['rzye'] = df['rzye'] / 1e+8
df['rzmre'] = df['rzmre'] / 1e+8
df['rzche'] = df['rzche'] / 1e+8
df['rqye'] = df['rqye'] / 1e+8
df['rzrqye'] = df['rzrqye'] / 1e+8
df = df.drop_duplicates()
df = df.reset_index(drop=True)
df = df.sort_values(by='date', ascending=True)
return df
def get_index_df(self, code, start_date, end_date):
cindex_client = CIndex(code)
df = cindex_client.get_k_data_in_range(start_date, end_date)
df['time'] = df.index.tolist()
df = df[[
'time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date'
]]
return df
def update(self, cdate=datetime.now().strftime('%Y-%m-%d')):
start_date = get_day_nday_ago(cdate, 100, dformat="%Y-%m-%d")
end_date = cdate
try:
#market info
sh_df = self.get_market_data(ct.SH_MARKET_SYMBOL, start_date,
end_date)
sz_df = self.get_market_data(ct.SZ_MARKET_SYMBOL, start_date,
end_date)
date_list = list(
set(sh_df.date.tolist()).intersection(set(
sz_df.date.tolist())))
sh_df = sh_df[sh_df.date.isin(date_list)]
sh_df = sh_df.reset_index(drop=True)
sz_df = sz_df[sz_df.date.isin(date_list)]
sz_df = sz_df.reset_index(drop=True)
#rzrq info
sh_rzrq_df = self.get_rzrq_info(ct.SH_MARKET_SYMBOL, start_date,
end_date)
sz_rzrq_df = self.get_rzrq_info(ct.SZ_MARKET_SYMBOL, start_date,
end_date)
date_list = list(
set(sh_rzrq_df.date.tolist()).intersection(
set(sz_rzrq_df.date.tolist())))
sh_rzrq_df = sh_rzrq_df[sh_rzrq_df.date.isin(date_list)]
sh_rzrq_df = sh_rzrq_df.reset_index(drop=True)
sz_rzrq_df = sz_rzrq_df[sz_rzrq_df.date.isin(date_list)]
sz_rzrq_df = sz_rzrq_df.reset_index(drop=True)
#average price info
av_df = self.get_index_df('880003', start_date, end_date)
#limit up and down info
limit_info = CLimit(self.dbinfo).get_data(cdate)
stock_info = self.rstock_client.get_data(cdate)
stock_info = stock_info[stock_info.volume >
0] #get volume > 0 stock list
stock_info = stock_info.reset_index(drop=True)
#index info
index_info = self.get_index_data(end_date)
#industry analysis
industry_info = self.get_industry_data(cdate)
#all stock info
all_stock_info = self.rstock_client.get_k_data_in_range(
start_date, end_date)
#gen review file and make dir for new data
self.doc.generate(cdate, sh_df, sz_df, sh_rzrq_df, sz_rzrq_df,
av_df, limit_info, stock_info, industry_info,
index_info, all_stock_info)
##gen review animation
#self.gen_animation()
except Exception as e:
self.logger.error(e)
traceback.print_exc()
def gen_animation(self, sfile=None):
style.use('fivethirtyeight')
Writer = animation.writers['ffmpeg']
writer = Writer(fps=1, metadata=dict(artist='biek'), bitrate=1800)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
_today = datetime.now().strftime('%Y-%m-%d')
cdata = self.mysql_client.get('select * from %s where date = "%s"' %
(ct.ANIMATION_INFO, _today))
if cdata is None: return None
cdata = cdata.reset_index(drop=True)
ctime_list = cdata.time.unique()
name_list = cdata.name.unique()
ctime_list = [
datetime.strptime(ctime, '%H:%M:%S') for ctime in ctime_list
]
frame_num = len(ctime_list)
if 0 == frame_num: return None
def animate(i):
ax.clear()
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S'))
ax.xaxis.set_major_locator(mdates.DayLocator())
ax.set_title('盯盘', fontproperties=get_chinese_font())
ax.set_xlabel('时间', fontproperties=get_chinese_font())
ax.set_ylabel('增长', fontproperties=get_chinese_font())
ax.set_ylim((-6, 6))
fig.autofmt_xdate()
for name in name_list:
pchange_list = list()
price_list = cdata[cdata.name == name]['price'].tolist()
pchange_list.append(0)
for _index in range(1, len(price_list)):
pchange_list.append(
10 * (price_list[_index] - price_list[_index - 1]) /
price_list[0])
ax.plot(ctime_list[0:i],
pchange_list[0:i],
label=name,
linewidth=1.5)
if pchange_list[i - 1] > 1 or pchange_list[i - 1] < -1:
ax.text(ctime_list[i - 1],
pchange_list[i - 1],
name,
font_properties=get_chinese_font())
ani = animation.FuncAnimation(fig,
animate,
frame_num,
interval=60000,
repeat=False)
sfile = '/data/animation/%s_animation.mp4' % _today if sfile is None else sfile
ani.save(sfile, writer)
plt.close(fig)
class DataManager:
def __init__(self, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.logger = getLogger(__name__)
self.index_objs = dict()
self.stock_objs = dict()
self.combination_objs = dict()
self.cal_client = CCalendar(dbinfo, redis_host)
self.index_info_client = IndexInfo()
self.comb_info_client = CombinationInfo(dbinfo, redis_host)
self.stock_info_client = CStockInfo(dbinfo, redis_host)
self.rindex_stock_data_client = RIndexStock(dbinfo, redis_host)
self.industry_info_client = IndustryInfo(dbinfo, redis_host)
self.rindustry_info_client = RIndexIndustryInfo(dbinfo, redis_host)
self.limit_client = CLimit(dbinfo, redis_host)
self.animation_client = CAnimation(dbinfo, redis_host)
self.subscriber = Subscriber()
self.quote_handler = StockQuoteHandler()
self.ticker_handler = TickerHandler()
self.connect_client = StockConnect(market_from=ct.SH_MARKET_SYMBOL,
market_to=ct.HK_MARKET_SYMBOL,
dbinfo=dbinfo,
redis_host=redis_host)
self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host)
self.emotion_client = Emotion(dbinfo=dbinfo, redis_host=redis_host)
self.sh_exchange_client = StockExchange(ct.SH_MARKET_SYMBOL)
self.sz_exchange_client = StockExchange(ct.SZ_MARKET_SYMBOL)
def is_collecting_time(self, now_time=datetime.now()):
_date = now_time.strftime('%Y-%m-%d')
y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3]
aft_open_hour, aft_open_minute, aft_open_second = (19, 00, 00)
aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute,
aft_open_second)
aft_close_hour, aft_close_minute, aft_close_second = (23, 59, 59)
aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute,
aft_close_second)
return aft_open_time < now_time < aft_close_time
def is_morning_time(self, now_time=datetime.now()):
_date = now_time.strftime('%Y-%m-%d')
y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3]
mor_open_hour, mor_open_minute, mor_open_second = (0, 0, 0)
mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute,
mor_open_second)
mor_close_hour, mor_close_minute, mor_close_second = (6, 30, 0)
mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute,
mor_close_second)
return mor_open_time < now_time < mor_close_time
def collect_combination_runtime_data(self):
def _combination_run(code_id):
self.combination_objs[code_id].run()
return (code_id, True)
todo_iplist = list(self.combination_objs.keys())
return concurrent_run(_combination_run, todo_iplist, num=10)
def collect_stock_runtime_data(self):
if self.ticker_handler.empty(): return
datas = self.ticker_handler.getQueue()
while not datas.empty():
df = datas.get()
df = df.set_index('time')
df.index = pd.to_datetime(df.index)
for code_str in set(df.code):
code_id = code_str.split('.')[1]
self.stock_objs[code_id].run(df.loc[df.code == code_str])
def init_real_stock_info(self):
concerned_list = self.comb_info_client.get_concerned_list()
prefix_concerned_list = [add_prifix(code) for code in concerned_list]
ret = self.subscriber.subscribe(prefix_concerned_list, SubType.TICKER,
self.ticker_handler)
if 0 == ret:
for code in concerned_list:
if code not in self.stock_objs:
self.stock_objs[code] = CStock(code,
self.dbinfo,
should_create_influxdb=True,
should_create_mysqldb=True)
return ret
def init_index_info(self):
index_list = ct.INDEX_DICT.keys()
prefix_index_list = [add_index_prefix(code) for code in index_list]
ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE,
self.quote_handler)
if 0 != ret:
self.logger.error("subscribe for index list failed")
return ret
for code in index_list:
if code not in self.index_objs:
self.index_objs[code] = CIndex(code,
should_create_influxdb=True,
should_create_mysqldb=True)
def collect_index_runtime_data(self):
if self.quote_handler.empty(): return
datas = self.quote_handler.getQueue()
while not datas.empty():
df = datas.get()
df['time'] = df.data_date + ' ' + df.data_time
df = df.drop(['data_date', 'data_time'], axis=1)
df = df.set_index('time')
df.index = pd.to_datetime(df.index)
for code_str in set(df.code):
code_id = code_str.split('.')[1]
self.index_objs[code_id].run(df.loc[df.code == code_str])
def run(self, sleep_time):
while True:
try:
if self.cal_client.is_trading_day():
if is_trading_time():
sleep_time = 1
if not self.subscriber.status():
self.subscriber.start()
if 0 == self.init_index_info(
) and 0 == self.init_real_stock_info():
self.init_combination_info()
else:
self.logger.debug("enter stop dict time")
self.subscriber.stop()
else:
self.collect_stock_runtime_data()
self.collect_combination_runtime_data()
self.collect_index_runtime_data()
self.animation_client.collect()
else:
sleep_time = 60
if self.subscriber.status():
self.subscriber.stop()
except Exception as e:
traceback.print_exc()
self.logger.error(e)
time.sleep(sleep_time)
def set_update_info(self,
step_length,
exec_date,
cdate=None,
filename=ct.STEPFILE):
step_info = dict()
if cdate is None: cdate = 'none'
step_info[cdate] = dict()
step_info[cdate]['step'] = step_length
step_info[cdate]['date'] = exec_date
with open(filename, 'w') as f:
json.dump(step_info, f)
self.logger.info("finish step :%s" % step_length)
def get_update_info(self,
cdate=None,
exec_date=None,
filename=ct.STEPFILE):
if cdate is None: cdate = 'none'
if not os.path.exists(filename): return (0, exec_date)
with open(filename, 'r') as f:
step_info = json.load(f)
if cdate not in step_info: return (0, exec_date)
return (step_info[cdate]['step'], step_info[cdate]['date'])
def bootstrap(self,
cdate=None,
exec_date=datetime.now().strftime('%Y-%m-%d')):
finished_step, exec_date = self.get_update_info(cdate, exec_date)
self.logger.info("enter updating.%s" % finished_step)
if finished_step < 1:
if not self.cal_client.init():
self.logger.error("cal_client init failed")
return False
self.set_update_info(1, exec_date, cdate)
if finished_step < 2:
if not self.index_info_client.update():
self.logger.error("index_info init failed")
return False
self.set_update_info(2, exec_date, cdate)
if finished_step < 3:
if not self.stock_info_client.update():
self.logger.error("stock_info init failed")
return False
self.set_update_info(3, exec_date, cdate)
if finished_step < 4:
if not self.comb_info_client.update():
self.logger.error("comb_info init failed")
return False
self.set_update_info(4, exec_date, cdate)
if finished_step < 5:
if not self.industry_info_client.update():
self.logger.error("industry_info init failed")
return False
self.set_update_info(5, exec_date, cdate)
if finished_step < 6:
if not self.download_and_extract(exec_date):
self.logger.error("download_and_extract failed")
return False
self.set_update_info(6, exec_date, cdate)
if finished_step < 7:
if not self.init_tdx_index_info(cdate):
self.logger.error("init_tdx_index_info failed")
return False
self.set_update_info(7, exec_date, cdate)
if finished_step < 8:
if not self.sh_exchange_client.update(exec_date, num=30):
self.logger.error("sh exchange update failed")
return False
self.set_update_info(8, exec_date, cdate)
if finished_step < 9:
if not self.sz_exchange_client.update(exec_date, num=30):
self.logger.error("sz exchange update failed")
return False
self.set_update_info(9, exec_date, cdate)
if finished_step < 10:
if not self.init_index_components_info(exec_date):
self.logger.error("init index components info failed")
return False
self.set_update_info(10, exec_date, cdate)
if finished_step < 11:
if not self.init_industry_info(cdate):
self.logger.error("init_industry_info failed")
return False
self.set_update_info(11, exec_date, cdate)
if finished_step < 12:
if not self.rindustry_info_client.update(exec_date):
self.logger.error("init %s rindustry info failed" % exec_date)
return False
self.set_update_info(12, exec_date, cdate)
if finished_step < 13:
if not self.limit_client.update(exec_date):
self.logger.error("init_limit_info failed")
return False
self.set_update_info(13, exec_date, cdate)
if finished_step < 14:
if not self.init_yesterday_hk_info(exec_date):
self.logger.error("init_yesterday_hk_info failed")
return False
self.set_update_info(14, exec_date, cdate)
if finished_step < 15:
if not self.margin_client.update(exec_date):
self.logger.error("init_yesterday_margin failed")
return False
self.set_update_info(15, exec_date, cdate)
if finished_step < 16:
if not self.init_stock_info(cdate):
self.logger.error("init_stock_info set failed")
return False
self.set_update_info(16, exec_date, cdate)
if finished_step < 17:
if not self.init_base_float_profit():
self.logger.error("init base float profit for all stock")
return False
self.set_update_info(17, exec_date, cdate)
if finished_step < 18:
if not self.rindex_stock_data_client.update(exec_date, num=300):
self.logger.error("rindex_stock_data set failed")
return False
self.set_update_info(18, exec_date, cdate)
self.logger.info("updating succeed")
return True
def update(self, sleep_time):
while True:
self.logger.info("enter daily update process. %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
try:
if self.cal_client.is_trading_day():
self.logger.info(
"is trading day. %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
if self.is_collecting_time():
self.logger.info(
"is collecting time. %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
self.bootstrap(
cdate=datetime.now().strftime('%Y-%m-%d'))
except Exception as e:
kill_process("google-chrome")
kill_process("renderer")
kill_process("Xvfb")
kill_process("zygote")
kill_process("defunct")
kill_process("show-component-extension-options")
self.logger.error(e)
time.sleep(sleep_time)
def init_combination_info(self):
trading_info = self.comb_info_client.get()
for _, code_id in trading_info['code'].iteritems():
if str(code_id) not in self.combination_objs:
self.combination_objs[str(code_id)] = Combination(
code_id, self.dbinfo)
def init_base_float_profit(self):
def _set_base_float_profit(code_id):
return (code_id,
True) if CStock(code_id).set_base_floating_profit() else (
code_id, False)
failed_list = self.stock_info_client.get().code.tolist()
return process_concurrent_run(_set_base_float_profit,
failed_list,
num=500)
def init_stock_info(self, cdate=None):
def _set_stock_info(_date, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, _date):
self.logger.info("%s set k data success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set k data failed" % code_id)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data exception:%s" % (code_id, e))
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=10, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_stock_info, mdate, bonus_info,
index_info)
if not process_concurrent_run(cfunc, failed_list, num=500):
succeed = False
return succeed
def init_industry_info(self, cdate):
def _set_industry_info(cdate, code_id):
return (code_id, CIndex(code_id).set_k_data(cdate))
df = self.industry_info_client.get()
if cdate is None:
cfunc = partial(_set_industry_info, cdate)
return concurrent_run(cfunc, df.code.tolist(), num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=30, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_industry_info, mdate)
if not concurrent_run(cfunc, df.code.tolist(), num=5):
succeed = False
return succeed
def init_yesterday_hk_info(self, cdate):
succeed = True
for data in ((ct.SH_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL),
(ct.SZ_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL)):
if not self.connect_client.set_market(data[0], data[1]):
self.logger.error("connect_client for %s failed" % data)
succeed = False
continue
if not self.connect_client.update(cdate):
succeed = False
self.connect_client.close()
self.connect_client.quit()
kill_process("zygote")
kill_process("defunct")
kill_process("show-component-extension-options")
return succeed
def init_index_components_info(self, cdate=None):
if cdate is None: cdate = datetime.now().strftime('%Y-%m-%d')
def _set_index_info(code_id):
_obj = self.index_objs[
code_id] if code_id in self.index_objs else CIndex(code_id)
return (code_id, _obj.set_components_data(cdate))
return concurrent_run(_set_index_info,
list(ct.INDEX_DICT.keys()),
num=10)
def init_tdx_index_info(self, cdate=None):
def _set_index_info(cdate, code_id):
try:
_obj = self.index_objs[
code_id] if code_id in self.index_objs else CIndex(code_id)
return (code_id, _obj.set_k_data(cdate))
except Exception as e:
self.logger.error(e)
return (code_id, False)
if cdate is None:
cfunc = partial(_set_index_info, cdate)
return concurrent_run(cfunc, list(ct.TDX_INDEX_DICT.keys()), num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=30, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_index_info, mdate)
if not concurrent_run(
cfunc, list(ct.TDX_INDEX_DICT.keys()), num=5):
succeed = False
return succeed
def download_and_extract(self, cdate):
try:
if not download(ct.ZIP_DIR, cdate): return False
list_files = os.listdir(ct.ZIP_DIR)
for filename in list_files:
if not filename.startswith('.'):
file_path = os.path.join(ct.ZIP_DIR, filename)
if os.path.exists(file_path):
unzip(file_path, ct.TIC_DIR)
return True
except Exception as e:
self.logger.error(e)
return False
class BullStockRatio:
def __init__(self, index_code, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.index_code = index_code
self.index_obj = CIndex(index_code,
dbinfo=self.dbinfo,
redis_host=redis_host)
self.db_name = self.index_obj.get_dbname(index_code)
self.logger = getLogger(__name__)
self.ris = RIndexStock(dbinfo, redis_host)
self.bull_stock_ratio_table = self.get_table_name()
self.redis = create_redis_obj(
) if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo,
dbname=self.db_name,
iredis=self.redis)
if not self.create(): raise Exception("create emotion table failed")
def delete(self):
self.mysql_client.delete(self.bull_stock_ratio_table)
def get_table_name(self):
return "%s_%s" % (self.db_name, ct.BULLSTOCKRATIO_TABLE)
def create(self):
if self.bull_stock_ratio_table not in self.mysql_client.get_all_tables(
):
sql = 'create table if not exists %s(date varchar(10) not null, ratio float, PRIMARY KEY (date))' % self.bull_stock_ratio_table
if not self.mysql_client.create(sql, self.bull_stock_ratio_table):
return False
return True
def is_date_exists(self, table_name, cdate):
if self.redis.exists(table_name):
return cdate in set(
str(tdate, encoding=ct.UTF8)
for tdate in self.redis.smembers(table_name))
return False
def get_k_data_between(self, start_date, end_date):
sql = "select * from %s where date between \"%s\" and \"%s\"" % (
self.get_table_name(), start_date, end_date)
return self.mysql_client.get(sql)
def get_components(self, cdate):
df = self.index_obj.get_components_data(cdate)
if df is None: return list()
if df.empty: return list()
if self.index_code == '000001': df = df[df.code.str.startswith('6')]
return df.code.tolist()
def get_data(self, cdate):
return self.ris.get_data(cdate)
def update(self, end_date=None, num=30):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
#start_date = "1997-12-30"
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
code_list = self.get_components(end_date)
if 0 == len(code_list):
self.logger.error("%s code_list for %s is empty" %
(end_date, self.index_code))
return False
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis=self.redis):
if not self.set_ratio(code_list, mdate):
self.logger.error("set %s score for %s set failed" %
(self.index_code, mdate))
succeed = False
return succeed
def get_profit_stocks(self, df):
data = df[df.profit >= 0]
return data.code.tolist()
def set_ratio(self,
now_code_list,
cdate=datetime.now().strftime('%Y-%m-%d')):
if self.is_date_exists(self.bull_stock_ratio_table, cdate):
self.logger.debug("existed date:%s, date:%s" %
(self.bull_stock_ratio_table, cdate))
return True
code_list = self.get_components(cdate)
if len(code_list) == 0: code_list = now_code_list
df = self.get_data(cdate)
if df is None: return False
if df.empty: return False
df = df[df.code.isin(code_list)]
if df.empty: return True
profit_code_list = self.get_profit_stocks(df)
bull_stock_num = len(profit_code_list)
bull_ration = 100 * bull_stock_num / len(df)
data = {'date': [cdate], 'ratio': [bull_ration]}
df = pd.DataFrame.from_dict(data)
if self.mysql_client.set(df, self.bull_stock_ratio_table):
return self.redis.sadd(self.bull_stock_ratio_table, cdate)
return False