def __init__(self, conf=None):
wx = lg.get_handle()
conf = os.path.dirname(os.path.abspath(sys.argv[0])) + "\\" + conf
# wx.info("[OBJ] Conf_handler : __init__ Read Conf : {}".format(conf))
self.rd = configparser.ConfigParser()
self.rd.read(conf)
def get_json_str(self, id, time_str=None, page_num=0):
wx = lg.get_handle()
if time_str is None or len(time_str) < 11:
wx.info("[RT_East][get_json_str] 时间段 不正确，退出")
return None
else:
[begin_time_str, end_time_str] = time_str.split("-")
begin_time_stamp = int(
time.mktime(
time.strptime(self.date_str + begin_time_str,
'%Y%m%d%H:%M')))
end_time_stamp = int(
time.mktime(
time.strptime(self.date_str + end_time_str,
'%Y%m%d%H:%M')))
my_timer = wx_timer(date_str='')
ret_zone = my_timer.tell_time_zone(t_stamp=end_time_stamp)
# 根据 end_time_stamp 获取 匹配的时间点，作为下次get_rt_data 的起始时间
record_stamp = ret_zone[2]
# 用于 rebase 函数
# 如果ID没有查询过，设置 rt_page_dict[id] = 0,下面每次循环完成后，rt_page_dict[id]累加1
# if id not in self.rt_page_dict.keys():
# self.rt_page_dict[id] = 0
# 用于 get_rt_data 函数，从文件读取 起始页面序号
if page_num != 0:
self.rt_page_dict[id] = page_num
# 检查 RT 对象是否已经获取 end_time_stamp 之前的交易数据
if id in self.rt_dict_df.keys(
) and self.rt_dict_df[id].time_stamp.max() >= end_time_stamp:
wx.info(
"[RT_East][{}] RT 对象已保存 [{}--{}]逐笔交易数据，目标时间段[{}--{}]不需要重新获取".
format(
id,
time.strftime(
"%H:%M:%S",
time.localtime(self.rt_dict_df[id].time_stamp.min())),
time.strftime(
"%H:%M:%S",
time.localtime(self.rt_dict_df[id].time_stamp.max())),
begin_time_str, end_time_str))
return None
market_code_dict = {
'60': ['1', '1'],
'00': ['2', '0'],
'30': ['2', '0'],
'68': ['1', '0']
}
while True:
url = "http://push2ex.eastmoney.com/getStockFenShi?pagesize="+self.item_page+\
"&ut=7eea3edcaed734bea9cbfc24409ed989&dpt=wzfscj&" \
"cb=jQuery1123021130998143685753_1580471904475&pageindex="+str(self.rt_page_dict[id])+\
"&id="+id+ market_code_dict[id[0:2]][0]+"&" \
"sort=1&ft=1&code="+id+"&market="+market_code_dict[id[0:2]][1]+"&_=1580471904476"
# sort =1 升序 ； 2 降序；
header = {
'Cookie':
'UM_distinctid=16bf36d52242f3-0693469a5596d3-e323069-1fa400-16bf36d5225362; _ntes_nnid=16b2182ff532e10833492eedde0996df,1563157161323; _ntes_nuid=16b2182ff532e10833492eedde0996df; vjuids=e0fb8aa0.16d4ee83324.0.e074eccb150e; [email protected]|1570190476|0|mail163|00&99|hen&1570190062&mail163#CN&null#10#0#0|&0|mail163|[email protected]; [email protected]:-1:1; mail_psc_fingerprint=8da65e9cc5769a658a69962d94f7c46f; _ntes_usstock_recent_=NTES%7C; _ntes_usstock_recent_=NTES%7C; vjlast=1568986903.1571018378.11; s_n_f_l_n3=e119c348b08890ac1571018378289; NNSSPID=0e35f22546f44023b00d65e2a3ca1f26; ne_analysis_trace_id=1571018721010; _ntes_stock_recent_=1002699%7C0600000%7C1000573; _ntes_stock_recent_=1002699%7C0600000%7C1000573; _ntes_stock_recent_=1002699%7C0600000%7C1000573; pgr_n_f_l_n3=e119c348b08890ac1571018815386610; vinfo_n_f_l_n3=e119c348b08890ac.1.5.1563157161368.1570632456351.1571018833379',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36',
'Referer':
'http://quote.eastmoney.com/f1.html?code=' + id + '&market=2'
}
# requests.packages.urllib3.disable_warnings()
http = urllib3.PoolManager()
try:
raw_data = http.request('GET', url, headers=header)
except Exception as e:
return None
finally:
if raw_data.status >= 300:
wx.info("Web response failed : {}".format(url))
return None
# 获得html源码,utf-8解码
str_type = chardet.detect(raw_data.data)
# unicode = raw_data.data.decode(str_type['encoding'])
unicode = lg.str_decode(raw_data.data, str_type['encoding'])
# 解析 JSON 字符串，并将实时交易数据保存到 RT 对象的rt_dict_df
[page_start_time_str,
page_end_time_str] = self.json_parse(id=id, json_str=unicode)
if page_start_time_str is not None:
page_end_time_stamp = int(
time.mktime(
time.strptime(self.date_str + page_end_time_str,
'%Y%m%d%H:%M:%S')))
page_start_time_stamp = int(
time.mktime(
time.strptime(self.date_str + page_start_time_str,
'%Y%m%d%H:%M:%S')))
# 找到 record_stamp 的页面，记录到 self.record_page_dict[id]
# 由get_rt_data 返回，并写入文件，作为下次 get_rt_data 读取的第一个页面序号
if time.strftime("%H:%M",
time.localtime(record_stamp)) != '13:00':
if page_end_time_stamp >= record_stamp and page_start_time_stamp <= record_stamp:
self.record_page_dict[id] = self.rt_page_dict[id]
else: # 遇到13:00 做特殊处理，11:30 之前所有页面的时间范围都不包含 13:00, 只需记录到最后一个页号
self.record_page_dict[id] = self.rt_page_dict[id]
if page_end_time_stamp >= end_time_stamp:
wx.info(
"[RT_East][{}] 第{}页 [{}--{}]逐笔交易数据，已获得目标时间段数据".format(
id, self.rt_page_dict[id], page_start_time_str,
page_end_time_str))
# 页数累加，rebase 再次调用get_json_str从此页号开始查询
self.rt_page_dict[id] += 1
break
else:
wx.info("[RT_East][{}] 第{}页 [{}--{}]逐笔交易数据, 未完成，继续获取下一页数据".
format(id, self.rt_page_dict[id],
page_start_time_str, page_end_time_str))
# 页数累加，rebase 再次调用get_json_str从此页号开始查询
self.rt_page_dict[id] += 1
time.sleep(0.5)
else:
wx.info("[RT_East] [{}] 第{}页 没有数据，退出".format(
id, self.rt_page_dict[id]))
break
return self.record_page_dict[id]
def rt_cmp_pa_baseline(self, rt=None, pa_bl_df=None):
wx = lg.get_handle()
rt_dict_df = rt.rt_dict_df
date_str = (date.today()).strftime('%Y%m%d')
if rt_dict_df is None:
wx.info("[Rt_Ana][RT_CMP_PA_Baseline] 实时数据字典 是空，退出")
return None
if pa_bl_df is None:
wx.info("[Rt_Ana][RT_CMP_PA_Baseline] 基线数据 是空，退出")
return None
cmp_pa_result = pd.DataFrame() # 保存最后的结果，并导入数据库
for id in rt_dict_df.keys():
if rt_dict_df[id] is None:
wx.info(
"[Rt_Ana][RT_CMP_PA_Baseline] {} 未产生实时交易数据，进入下一支股票".format(
id))
continue
# 起始时间边界对齐
[frame_begin_stamp,
frame_begin_time_str] = self.rt_df_find_start_stamp(
rt_stamp=rt_dict_df[id]['time_stamp'].min())
end_stamp = rt_dict_df[id].time_stamp.max()
# 按时间段 切片rt数据，计算PA向量长度、角度、涨幅、成交金额，然后对比基线数据
while frame_begin_stamp < end_stamp:
# 每个时间段，初始化一次 rt_pa_df ，保存该时间段内的RT resample数据
rt_pa_df = pd.DataFrame()
# frame_begin_time_str = time.strftime("%H:%M", time.localtime(frame_begin_stamp))
frame_end_time_str = self.t_frame_dict.get(
frame_begin_time_str)[0]
if frame_end_time_str is None:
wx.info(
"[Rt_Ana][RT_CMP_PA_Baseline] {} [{}] 起始时间不属于正常范围！！！！".
format(id, frame_begin_time_str))
break
t_frame = frame_begin_time_str + "-" + frame_end_time_str
frame_end_stamp = int(
time.mktime(
time.strptime(date_str + frame_end_time_str,
'%Y%m%d%H:%M')))
# PA 实时数据 与 基线的对比，不需要 整个时间段，随时可以对照 基线的 t_frame 数据
# if frame_end_stamp > end_stamp:
# wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] {} {} 已超出本次获取的实时数据范围，进入下一支股票".format(id, t_frame))
# break
# 从基线DataFrame 提取该id 对应 t_frame 的基线数据
id_bl_df = pa_bl_df.loc[(pa_bl_df['id'] == id) &
(pa_bl_df['t_frame'] == t_frame), ]
# 从RT DataFrame 提取该 id 对应
if frame_end_time_str == '15:00': # 15:00 收市后，有最后一笔交易记录 产生在 15:00 之后若干秒
rt_df = rt.rt_dict_df[id].loc[(
rt.rt_dict_df[id]['time_stamp'] >=
frame_begin_stamp)].copy()
else:
rt_df = rt.rt_dict_df[id].loc[
(rt.rt_dict_df[id]['time_stamp'] >= frame_begin_stamp)
& (rt.rt_dict_df[id]['time_stamp'] < frame_end_stamp
)].copy()
if rt_df is None or rt_df.empty:
wx.info(
"[Rt_Ana][RT_CMP_PA_Baseline] [{}] 在[{}]期间交易数据为空，开始处理下一支股票"
.format(id, t_frame))
break
#
# 开始resample RT数据，并计算 PA 向量、角度
#
rt_df = rt_df.sort_values(by="time_stamp", ascending=True)
rt_df['amount'] = rt_df['price'] * rt_df['vol']
rt_df['pd_time'] = date_str + " " + rt_df['time_str']
rt_df['pd_time'] = pd.to_datetime(rt_df['pd_time'],
format="%Y%m%d %H:%M:%S")
rt_df.set_index('pd_time', inplace=True)
rt_df['price'] = pd.to_numeric(rt_df['price'])
rt_pa_df['amount'] = rt_df['amount'].resample(
self.rt_PA_resample_secs).sum()
rt_pa_df['min_price'] = rt_df['price'].resample(
self.rt_PA_resample_secs).min()
rt_pa_df['max_price'] = rt_df['price'].resample(
self.rt_PA_resample_secs).max()
rt_pa_df['pct_chg'] = rt_pa_df['max_price'] / rt_pa_df[
'min_price'] - 1
rt_pa_df['pct_chg_enhanced'] = (
rt_pa_df['max_price'] / rt_pa_df['min_price'] -
1) * 500000000
# Time(High Price) - Time (Low Price) > 0 上涨； >0 下跌, （高价时间 - 低价时间）
rt_pa_df['pct_up_down'] = rt_df['price'].resample(
self.rt_PA_resample_secs).apply(
lambda x: x.idxmax() - x.idxmin() if len(x) > 0 else
(pd.to_datetime(0) - pd.to_datetime(0)))
rt_pa_df.fillna(0, inplace=True)
# 過濾掉 成交量 == 0 & 價格變動 ==0 的 時間段記錄
rt_pa_df = rt_pa_df.loc[(rt_pa_df['amount'] > 0) &
(rt_pa_df['pct_chg_enhanced'] > 0), ]
# 将 pd.datetime64 之差 转换成 float 类型，方便判断 时间切片内的涨跌
rt_pa_df['pct_up_down'] = pd.to_numeric(
rt_pa_df['pct_up_down'])
# 量价向量长度
rt_pa_df['pa_vector'] = pow(
pow(rt_pa_df['amount'], 2) +
pow(rt_pa_df['pct_chg_enhanced'], 2), 0.5)
# 量价向量方向
rt_pa_df['pct_dir'] = rt_pa_df['pct_up_down'].apply(
lambda x: x / abs(x) if x != 0 else 0)
# 量价向量角度, X轴 涨跌幅度 ， Y轴 成交金额
# 数值越小：小金额，大幅度涨跌
# 数值越大：大金额，小幅度涨跌
rt_pa_df['pa_angle'] = rt_pa_df['amount'] / rt_pa_df[
'pct_chg_enhanced'] * rt_pa_df['pct_dir']
rt_pa_df['id'] = id
rt_pa_df.reset_index(drop=False, inplace=True)
rt_pa_df['pd_time'] = rt_pa_df['pd_time'].dt.strftime(
'%Y%m%d %H:%M:%S')
rt_pa_df['date'] = rt_pa_df['pd_time'].apply(lambda x: x[0:8])
rt_pa_df['time'] = rt_pa_df['pd_time'].apply(lambda x: x[9:])
pa_cmp_df = pd.merge(rt_pa_df, id_bl_df, on=['id'], how='left')
pa_cmp_df['pct_chg'] *= 100
pa_cmp_df['b_up_pct_max'] *= 100
pa_cmp_df['b_up_pct_min'] *= 100
pa_cmp_df['b_down_pct_max'] *= 100
pa_cmp_df['b_down_pct_min'] *= 100
# 按照上涨、下跌分类
# 开始与基线数据比对
cmp_up_dict = {
'pa_vector': [
'b_up_pa_max', 'b_up_pa_min', 'PA_UP长度-[超高]-',
'PA_UP长度-[超低]-', '[PA_UP向量超长]', '[PA_UP向量超短'
], # PA向量长度
'pa_angle': [
'b_up_ang_max', 'b_up_ang_min', 'PA_UP角度-[量大幅小]-',
'PA_UP角度-[量小幅大]-', '[PA_UP量大幅小]', '[PA_UP量小幅大]'
], # PA向量角度
# 'amount': ['b_up_amount_max', 'b_up_amount_min', '大买单金额占比-[超高]-', '大买单金额占比-[超低]-','[B多]大买单净比高', '[B空]大买单净比低'], # 大单买入金额占比
# 'pct_chg': ['b_up_pct_max', 'b_up_pct_min', '大卖单金额占比-[超高]-', '大卖单金额占比-[超低]-', '[B空]大卖单净比高', '[B多]大卖单净比低'], # 大单卖出金额占比
}
cmp_up_result_df = pd.DataFrame()
# 筛选出上涨的Dataframe
pa_up_cmp_df = pa_cmp_df.loc[pa_cmp_df['pct_dir'] > 0, ]
if pa_up_cmp_df is not None and len(pa_up_cmp_df) > 0:
for key in cmp_up_dict.keys():
high_df = pa_up_cmp_df.loc[
pa_up_cmp_df[key] > pa_up_cmp_df[cmp_up_dict[key]
[0]]]
high_df = self._cmp_data_process_2_(
df=high_df,
key=key,
val=cmp_up_dict[key][0],
msg=cmp_up_dict[key][2],
type=cmp_up_dict[key][4])
if key == 'pa_vector': # PA 向量长度短，不做异常记录
low_df = None
else:
low_df = pa_up_cmp_df.loc[
pa_up_cmp_df[key] < pa_up_cmp_df[
cmp_up_dict[key][1]]]
low_df = self._cmp_data_process_2_(
df=low_df,
key=key,
val=cmp_up_dict[key][1],
msg=cmp_up_dict[key][3],
type=cmp_up_dict[key][5])
if cmp_up_result_df is None or len(
cmp_up_result_df) == 0:
if high_df is not None:
cmp_up_result_df = high_df
if low_df is not None:
cmp_up_result_df = cmp_up_result_df.append(
low_df)
else:
cmp_up_result_df = low_df
else:
if high_df is not None:
cmp_up_result_df = cmp_up_result_df.append(
high_df)
if low_df is not None:
cmp_up_result_df = cmp_up_result_df.append(
low_df)
# 按照上涨、下跌分类
# 开始与基线数据比对
cmp_down_dict = {
'pa_vector': [
'b_down_pa_max', 'b_down_pa_min', 'PA_DOWN长度-[超高]-',
'PA_DOWN长度-[超低]-', '[PA_DOWN向量超长]', '[PA_DOWN向量超短'
], # PA向量长度
'pa_angle': [
'b_down_ang_max', 'b_down_ang_min',
'PA_DOWN角度-[量小幅大]-', 'PA_DOWN角度-[量大幅小]-',
'[PA_DOWN量小幅大]', '[PA_DOWN量大幅小'
], # PA向量角度
# 'amount': ['b_up_amount_max', 'b_up_amount_min', '大买单金额占比-[超高]-', '大买单金额占比-[超低]-','[B多]大买单净比高', '[B空]大买单净比低'], # 大单买入金额占比
# 'pct_chg': ['b_up_pct_max', 'b_up_pct_min', '大卖单金额占比-[超高]-', '大卖单金额占比-[超低]-', '[B空]大卖单净比高', '[B多]大卖单净比低'], # 大单卖出金额占比
}
cmp_down_result_df = pd.DataFrame()
# 筛选出下跌的Dataframe
pa_down_cmp_df = pa_cmp_df.loc[pa_cmp_df['pct_dir'] < 0, ]
if pa_down_cmp_df is not None and len(pa_down_cmp_df) > 0:
for key in cmp_down_dict.keys():
high_df = pa_down_cmp_df.loc[
pa_down_cmp_df[key] > pa_down_cmp_df[
cmp_down_dict[key][0]]]
high_df = self._cmp_data_process_2_(
df=high_df,
key=key,
val=cmp_down_dict[key][0],
msg=cmp_down_dict[key][2],
type=cmp_down_dict[key][4])
if key == 'pa_vector':
low_df = None # PA 向量长度短，不做异常记录
else:
low_df = pa_down_cmp_df.loc[
pa_down_cmp_df[key] < pa_down_cmp_df[
cmp_down_dict[key][1]]]
low_df = self._cmp_data_process_2_(
df=low_df,
key=key,
val=cmp_down_dict[key][1],
msg=cmp_down_dict[key][3],
type=cmp_down_dict[key][5])
if cmp_down_result_df is None or len(
cmp_down_result_df) == 0:
if high_df is not None:
cmp_down_result_df = high_df
if low_df is not None:
cmp_down_result_df = cmp_down_result_df.append(
low_df)
else:
cmp_down_result_df = low_df
else:
if high_df is not None:
cmp_down_result_df = cmp_down_result_df.append(
high_df)
if low_df is not None:
cmp_down_result_df = cmp_down_result_df.append(
low_df)
# 本ID 的异常PA检测完成，收集结果到 cmp_pa_result
if cmp_pa_result is None or len(cmp_pa_result) == 0:
if cmp_up_result_df is not None:
cmp_pa_result = cmp_up_result_df
if cmp_down_result_df is not None:
cmp_pa_result = cmp_pa_result.append(
cmp_down_result_df)
else:
cmp_pa_result = cmp_down_result_df
else:
if cmp_up_result_df is not None:
cmp_pa_result = cmp_pa_result.append(cmp_up_result_df)
if cmp_down_result_df is not None:
cmp_pa_result = cmp_pa_result.append(
cmp_down_result_df)
# 准备进入下一个 while frame_begin_stamp < end_stamp 循环
frame_begin_time_str = self.t_frame_dict.get(
frame_begin_time_str)[1]
if len(frame_begin_time_str) == 0:
wx.info("[Rt_Ana][RT_CMP_PA_Baseline] {} {} 已处理完毕，进入下一支股票".
format(id, t_frame))
break
frame_begin_stamp = int(
time.mktime(
time.strptime(date_str + frame_begin_time_str,
'%Y%m%d%H:%M')))
cols = ['id', 'date', 'time', 'type', 'msg']
cmp_pa_result = cmp_pa_result.loc[:, cols]
return cmp_pa_result
def calc_arr(self,
pre_id='',
fresh=False,
data_src='cq',
bt_start_date='',
bt_end_date=''):
wx = lg.get_handle()
if data_src == 'cq':
if re.match('002', pre_id) is not None:
t_name = self.cq_tname_002
elif re.match('00', pre_id) is not None:
t_name = self.cq_tname_00
elif re.match('30', pre_id) is not None:
t_name = self.cq_tname_30
elif re.match('60', pre_id) is not None:
t_name = self.cq_tname_60
elif re.match('68', pre_id) is not None:
t_name = self.cq_tname_68
else:
wx.info(
"[Class MA_kits: calc] failed to identify the Stock_id {}".
format(pre_id))
return None
elif data_src == 'qfq':
if re.match('002', pre_id) is not None:
t_name = self.qfq_tname_002
elif re.match('00', pre_id) is not None:
t_name = self.qfq_tname_00
elif re.match('30', pre_id) is not None:
t_name = self.qfq_tname_30
elif re.match('60', pre_id) is not None:
t_name = self.qfq_tname_60
elif re.match('68', pre_id) is not None:
t_name = self.qfq_tname_68
else:
wx.info(
"[Class MA_kits: calc] failed to identify the Stock_id {}".
format(pre_id))
return None
elif data_src == 'bt_qfq':
if re.match('002', pre_id) is not None:
t_name = self.bt_qfq_tname_002
elif re.match('00', pre_id) is not None:
t_name = self.bt_qfq_tname_00
elif re.match('30', pre_id) is not None:
t_name = self.bt_qfq_tname_30
elif re.match('60', pre_id) is not None:
t_name = self.bt_qfq_tname_60
elif re.match('68', pre_id) is not None:
t_name = self.bt_qfq_tname_68
else:
wx.info(
"[Class MA_kits: calc] failed to identify the Stock_id {}".
format(pre_id))
return None
else:
wx.info("[Class MA_kits: calc] failed to identify the Data Src {}".
format(data_src))
return None
# today = datetime.now().strftime('%Y%m%d')
if data_src == 'bt_qfq':
sql = "select id, date, close from " + t_name + " where close > 0 and date between " \
+ bt_start_date + " and "+bt_end_date+" order by id"
else:
start_date = (datetime.now() + timedelta(days=-480)).strftime(
'%Y%m%d') # 起始日期 为记录日期+1天
sql = "select id, date, close from " + t_name + " where close > 0 and date >= " + start_date + " order by id"
df_ma = self.db._exec_sql(sql)
# df_ma.sort_values(by='date', ascending=True, inplace=True)
# df_ma.fillna(0, inplace=True)
# df_grouped = df_ma['close'].groupby(df_ma['id'])
if df_ma is None or df_ma.empty:
return None
df_tmp = pd.DataFrame()
for duration in self.ma_duration:
df_tmp['MA_' + duration] = df_ma['close'].groupby(
df_ma['id']).rolling(int(duration)).mean()
# 整理 移动均值数据，合并DataFrame
df_tmp.reset_index(drop=True, inplace=True)
df_ma = pd.merge(df_ma,
df_tmp,
left_index=True,
right_index=True,
how='inner')
# EMA 12 26 指数移动均值
# df_grouped = df_ma['close'].groupby(df_ma['id'])
df_tmp = pd.DataFrame()
for duration in self.ema_duration:
# df_tmp['EMA_' + duration] = df_grouped['close'].ewm(span=int(duration)).mean()
df_tmp['EMA_' + duration] = df_ma['close'].groupby(
df_ma['id']).apply(lambda x: x.ewm(span=int(duration)).mean())
# 整理 指数移动均值数据，合并DataFrame
df_tmp.reset_index(drop=True, inplace=True)
df_ma = pd.merge(df_ma,
df_tmp,
left_index=True,
right_index=True,
how='inner')
# MACD 快线
df_ma['DIF'] = df_ma['EMA_' + self.ema_duration[0]] - df_ma[
'EMA_' + self.ema_duration[1]]
# MACD 慢线
df_tmp = pd.DataFrame()
df_tmp['DEA'] = df_ma['DIF'].groupby(
df_ma['id']).apply(lambda x: x.ewm(span=9).mean())
df_ma = pd.merge(df_ma,
df_tmp,
left_index=True,
right_index=True,
how='inner')
# Bolling 20 计算
df_tmp = pd.DataFrame()
df_ma['bolling_mid'] = df_ma['MA_' + self.bolling]
df_tmp['tmp2'] = df_ma['close'].groupby(df_ma['id']).rolling(
int(self.bolling)).std()
df_tmp.reset_index(drop=True, inplace=True)
df_ma = pd.merge(df_ma,
df_tmp,
left_index=True,
right_index=True,
how='inner')
df_ma['bolling_top'] = df_ma['MA_' + self.bolling] + 2 * df_ma['tmp2']
df_ma['bolling_bottom'] = df_ma['MA_' +
self.bolling] - 2 * df_ma['tmp2']
df_ma.drop(columns=['tmp2'], inplace=True)
df_ma.drop(columns=['close'], inplace=True)
df_ma.fillna(0, inplace=True)
# df_ma.dropna(axis=0, how="any", inplace=True)
if (fresh == False):
df_ma[['date']] = df_ma[['date']].apply(pd.to_numeric)
df_ret = df_ma.iloc[df_ma.groupby(
['id']).apply(lambda x: x['date'].idxmax())]
return df_ret
return df_ma
def __del__(self):
wx = lg.get_handle()
self.db.cursor.close()
self.db.handle.close()
wx.info("[OBJ] analyzer __del__() called")
def db_load_into_daily_data(self,
dd_df=None,
pre_id='',
mode='basic',
type='cq'):
wx = lg.get_handle()
if (type == 'cq'):
if pre_id == '00%':
t_name = self.dd_cq_00
elif pre_id == '30%':
t_name = self.dd_cq_30
elif pre_id == '60%':
t_name = self.dd_cq_60
elif pre_id == '002%':
t_name = self.dd_cq_002
elif pre_id == '68%':
t_name = self.dd_cq_68
else:
wx.info(
"[db_load_into_daily_data]: TYPE = cq, pre_id ( {} )is NOT Match"
.format(pre_id))
return None
elif (type == 'qfq'):
if pre_id == '00%':
t_name = self.dd_qfq_00
elif pre_id == '30%':
t_name = self.dd_qfq_30
elif pre_id == '60%':
t_name = self.dd_qfq_60
elif pre_id == '002%':
t_name = self.dd_qfq_002
elif pre_id == '68%':
t_name = self.dd_qfq_68
else:
wx.info(
"[db_load_into_daily_data]: TYPE = qfq, pre_id ( {} )is NOT Match"
.format(pre_id))
return None
elif (type == 'bt_qfq'):
if re.match('002', pre_id) is not None:
t_name = self.bt_dd_qfq_002
elif re.match('30', pre_id) is not None:
t_name = self.bt_dd_qfq_30
elif re.match('60', pre_id) is not None:
t_name = self.bt_dd_qfq_60
elif re.match('00', pre_id) is not None:
t_name = self.bt_dd_qfq_00
elif re.match('68', pre_id) is not None:
t_name = self.bt_dd_qfq_68
else:
wx.info(
"[db_load_into_daily_data]: TYPE = bt_qfq, pre_id ( {} )is NOT Match"
.format(pre_id))
return None
else:
wx.info("[db_load_into_daily_data]: TYPE ( {} ) is NOT Match")
return None
if dd_df is None or dd_df.empty or t_name is None:
wx.info(
"[db_load_into_daily_data] Err: Daily Data Frame or Table Name is Empty,"
)
return -1
dd_array = dd_df.values.tolist()
i = 0
while i < len(dd_array):
dd_array[i] = tuple(dd_array[i])
i += 1
if mode == 'full':
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, open=%s, high=%s, low=%s, " \
"close=%s, pre_close=%s, chg=%s, pct_chg=%s,vol=%s, amount=%s, " \
"qrr=%s, tor=%s, pct_up_down=%s, pe=%s, pb=%s"
elif mode == 'basic':
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, open=%s, high=%s, low=%s, " \
"close=%s, pre_close=%s, chg=%s, pct_chg=%s,vol=%s, amount=%s"
else:
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, open=%s, high=%s, low=%s, " \
"close=%s, pre_close=%s, chg=%s, pct_chg=%s,vol=%s, amount=%s"
i_scale = 1000
for i in range(0, len(dd_array), i_scale):
tmp_array = dd_array[i:i + i_scale]
wx.info(
"[db_load_into_daily_data][{}][{}] Loaded {} ~ {} , total {} ".
format(type, t_name, i, i + i_scale, len(dd_array)))
self.db.cursor.executemany(sql, tmp_array)
self.db.handle.commit()
def get_baseline_big_deal(self, days=3):
wx = lg.get_handle()
# 量价向量 的取样resample的时间段
std_times = float(self.h_conf.rd_opt('rt_analysis_rules', 'big_deal_std_times'))
bl_df = self.db.get_bl_big_deal(days=days)
baseline_bd_df = pd.DataFrame()
for tmp_df in bl_df.groupby(bl_df['id']):
each_id_df = tmp_df[1].fillna(0, inplace=False)
b_qty_ave = each_id_df['big_qty'].groupby(each_id_df['t_frame']).mean()
b_qty_std = each_id_df['big_qty'].groupby(each_id_df['t_frame']).std().fillna(0)
b_qty_max = b_qty_ave + b_qty_std * std_times
b_qty_min = b_qty_ave - b_qty_std * std_times
b_qty_max.name = 'b_qty_max'
b_qty_min.name = 'b_qty_min'
b_abs_pct_ave = each_id_df['big_abs_pct'].groupby(each_id_df['t_frame']).mean()
b_abs_pct_std = each_id_df['big_abs_pct'].groupby(each_id_df['t_frame']).std().fillna(0)
b_pct_max = b_abs_pct_ave + b_abs_pct_std * std_times
b_pct_min = b_abs_pct_ave - b_abs_pct_std * std_times
b_pct_max.name = 'b_pct_max'
b_pct_min.name = 'b_pct_min'
b_buy_pct_ave = each_id_df['big_buy_pct'].groupby(each_id_df['t_frame']).mean()
b_buy_pct_std = each_id_df['big_buy_pct'].groupby(each_id_df['t_frame']).std().fillna(0)
b_buy_pct_max = b_buy_pct_ave + b_buy_pct_std * std_times
b_buy_pct_min = b_buy_pct_ave - b_buy_pct_std * std_times
b_buy_pct_max.name = 'b_buy_pct_max'
b_buy_pct_min.name = 'b_buy_pct_min'
b_sell_pct_ave = each_id_df['big_sell_pct'].groupby(each_id_df['t_frame']).mean()
b_sell_pct_std = each_id_df['big_sell_pct'].groupby(each_id_df['t_frame']).std().fillna(0)
b_sell_pct_max = b_sell_pct_ave + b_sell_pct_std * std_times
b_sell_pct_min = b_sell_pct_ave - b_sell_pct_std * std_times
b_sell_pct_max.name = 'b_sell_pct_max'
b_sell_pct_min.name = 'b_sell_pct_min'
each_id_df = each_id_df[~(each_id_df['buy_qty'].isin([0]))]
all_buy_qty_ave = each_id_df['buy_qty'].groupby(each_id_df['t_frame']).mean()
all_buy_qty_std = each_id_df['buy_qty'].groupby(each_id_df['t_frame']).std().fillna(0)
all_buy_qty_max = all_buy_qty_ave + all_buy_qty_std * std_times
all_buy_qty_min = all_buy_qty_ave - all_buy_qty_std * std_times
all_buy_qty_max.name = 'all_buy_qty_max'
all_buy_qty_min.name = 'all_buy_qty_min'
each_id_df = each_id_df[~(each_id_df['buy_amount'].isin([0]))]
all_buy_amount_ave = each_id_df['buy_amount'].groupby(each_id_df['t_frame']).mean()
all_buy_amount_std = each_id_df['buy_amount'].groupby(each_id_df['t_frame']).std().fillna(0)
all_buy_amount_max = all_buy_amount_ave + all_buy_amount_std * std_times
all_buy_amount_min = all_buy_amount_ave - all_buy_amount_std * std_times
all_buy_amount_max.name = 'all_buy_amount_max'
all_buy_amount_min.name = 'all_buy_amount_min'
each_id_df = each_id_df[~(each_id_df['sell_qty'].isin([0]))]
all_sell_qty_ave = each_id_df['sell_qty'].groupby(each_id_df['t_frame']).mean()
all_sell_qty_std = each_id_df['sell_qty'].groupby(each_id_df['t_frame']).std().fillna(0)
all_sell_qty_max = all_sell_qty_ave + all_sell_qty_std * std_times
all_sell_qty_min = all_sell_qty_ave - all_sell_qty_std * std_times
all_sell_qty_max.name = 'all_sell_qty_max'
all_sell_qty_min.name = 'all_sell_qty_min'
each_id_df = each_id_df[~(each_id_df['sell_amount'].isin([0]))]
all_sell_amount_ave = each_id_df['sell_amount'].groupby(each_id_df['t_frame']).mean()
all_sell_amount_std = each_id_df['sell_amount'].groupby(each_id_df['t_frame']).std().fillna(0)
all_sell_amount_max = all_sell_amount_ave + all_sell_amount_std * std_times
all_sell_amount_min = all_sell_amount_ave - all_sell_amount_std * std_times
all_sell_amount_max.name = 'all_sell_amount_max'
all_sell_amount_min.name = 'all_sell_amount_min'
id_df = pd.concat([b_qty_max, b_qty_min, b_pct_max, b_pct_min, b_buy_pct_max, b_buy_pct_min,
b_sell_pct_max, b_sell_pct_min, all_buy_qty_max, all_buy_qty_min,
all_buy_amount_max, all_buy_amount_min, all_sell_qty_max, all_sell_qty_min,
all_sell_amount_max, all_sell_amount_min]
, axis=1, sort=False)
# cols = ['qty_max', 'qty_min' , 'pct_max', 'pct_min', 'buy_pct_max', 'buy_pct_min',
# 'sell_pct_max', 'sell_pct_min', 'all_buy_qty_max', 'all_buy_qty_min',
# 'all_buy_amount_max', 'all_buy_amount_min', 'all_sell_qty_max', 'all_sell_qty_min',
# 'all_sell_amount_max', 'all_sell_amount_min']
# id_df = id_df.loc[:,cols]
id_df['id']=tmp_df[0]
if baseline_bd_df is None or baseline_bd_df.empty:
baseline_bd_df = id_df
else:
baseline_bd_df = baseline_bd_df.append(id_df)
baseline_bd_df.reset_index(drop=False, inplace=True)
baseline_bd_df.rename(columns={'index':'t_frame'}, inplace=True)
return baseline_bd_df
def __init__(self,
f_conf='',
f_start_date='',
f_end_date='',
data_src='qfq'):
wx = lg.get_handle()
try:
self.f_conf = conf_handler(conf=f_conf)
self.h_conf = conf_handler(conf="stock_analyer.conf")
self.pe = self.f_conf.rd_opt('filter_fix', 'pe')
self.total_amount = self.f_conf.rd_opt('filter_fix',
'total_amount')
self.high_price = self.f_conf.rd_opt('filter_fix', 'high_price')
self.days = self.f_conf.rd_opt('filter_fix', 'below_ma55_days')
# self.filter_growth_below_pct = self.f_conf.rd_opt('filter_fix', 'filter_growth_below_pct')
# self.filter_high_left_power_request = self.f_conf.rd_opt('filter_fix', 'filter_high_left_power_request')
# self.filter_high_right_power_request = self.f_conf.rd_opt('filter_fix', 'filter_high_right_power_request')
# self.filter_cur_left_power_request = self.f_conf.rd_opt('filter_fix', 'filter_cur_left_power_request')
# self.filter_golden_pct = self.f_conf.rd_opt('filter_fix', 'filter_golden_pct')
# self.filter_golden_pct_request = float(self.f_conf.rd_opt('filter_fix', 'filter_golden_pct_request'))
self.daily_cq_t_00 = self.h_conf.rd_opt('db', 'daily_table_cq_00')
self.daily_cq_t_30 = self.h_conf.rd_opt('db', 'daily_table_cq_30')
self.daily_cq_t_60 = self.h_conf.rd_opt('db', 'daily_table_cq_60')
self.daily_cq_t_68 = self.h_conf.rd_opt('db', 'daily_table_cq_68')
self.daily_cq_t_002 = self.h_conf.rd_opt('db',
'daily_table_cq_002')
self.daily_qfq_t_00 = self.h_conf.rd_opt('db',
'daily_table_qfq_00')
self.daily_qfq_t_30 = self.h_conf.rd_opt('db',
'daily_table_qfq_30')
self.daily_qfq_t_60 = self.h_conf.rd_opt('db',
'daily_table_qfq_60')
self.daily_qfq_t_68 = self.h_conf.rd_opt('db',
'daily_table_qfq_68')
self.daily_qfq_t_002 = self.h_conf.rd_opt('db',
'daily_table_qfq_002')
self.bt_daily_qfq_t_00 = self.h_conf.rd_opt(
'db', 'bt_daily_table_qfq_00')
self.bt_daily_qfq_t_30 = self.h_conf.rd_opt(
'db', 'bt_daily_table_qfq_30')
self.bt_daily_qfq_t_60 = self.h_conf.rd_opt(
'db', 'bt_daily_table_qfq_60')
self.bt_daily_qfq_t_68 = self.h_conf.rd_opt(
'db', 'bt_daily_table_qfq_68')
self.bt_daily_qfq_t_002 = self.h_conf.rd_opt(
'db', 'bt_daily_table_qfq_002')
self.ma_cq_table_60 = self.h_conf.rd_opt('db', 'ma_cq_table_60')
self.ma_cq_table_30 = self.h_conf.rd_opt('db', 'ma_cq_table_30')
self.ma_cq_table_00 = self.h_conf.rd_opt('db', 'ma_cq_table_00')
self.ma_cq_table_002 = self.h_conf.rd_opt('db', 'ma_cq_table_002')
self.ma_cq_table_68 = self.h_conf.rd_opt('db', 'ma_cq_table_68')
self.ma_qfq_table_60 = self.h_conf.rd_opt('db', 'ma_qfq_table_60')
self.ma_qfq_table_30 = self.h_conf.rd_opt('db', 'ma_qfq_table_30')
self.ma_qfq_table_00 = self.h_conf.rd_opt('db', 'ma_qfq_table_00')
self.ma_qfq_table_002 = self.h_conf.rd_opt('db',
'ma_qfq_table_002')
self.ma_qfq_table_68 = self.h_conf.rd_opt('db', 'ma_qfq_table_68')
self.ma_bt_qfq_table_00 = self.h_conf.rd_opt(
'db', 'ma_bt_qfq_table_00')
self.ma_bt_qfq_table_30 = self.h_conf.rd_opt(
'db', 'ma_bt_qfq_table_30')
self.ma_bt_qfq_table_60 = self.h_conf.rd_opt(
'db', 'ma_bt_qfq_table_30')
self.ma_bt_qfq_table_68 = self.h_conf.rd_opt(
'db', 'ma_bt_qfq_table_30')
self.ma_bt_qfq_table_002 = self.h_conf.rd_opt(
'db', 'ma_bt_qfq_table_30')
host = self.h_conf.rd_opt('db', 'host')
database = self.h_conf.rd_opt('db', 'database')
user = self.h_conf.rd_opt('db', 'user')
pwd = self.h_conf.rd_opt('db', 'pwd')
self.db = db_ops(host=host, db=database, user=user, pwd=pwd)
wx.info("[OBJ] filter_fix : __init__ called")
# 指定日期，用于回测，默认日期是当前最近交易日
if data_src == 'cq' or data_src == 'qfq':
sql = "SELECT date from " + self.daily_cq_t_00 + " order by date desc limit 1"
df_date = self.db._exec_sql(sql=sql)
self.f_end_date = df_date.iloc[0, 0]
self.f_start_date = (date.today() +
timedelta(days=-240)).strftime('%Y%m%d')
elif data_src == 'bt_qfq':
if f_end_date == '' or f_start_date == '':
wx.info("[Filter_fix]回测未设置起止日期区间[{}-{}]".format(
f_start_date, f_end_date))
return
self.f_end_date = f_end_date
self.f_start_date = f_start_date
self.data_src = data_src
except Exception as e:
raise e
def set_baseline_big_deal(self, rt=None , date_str='', time_frame_arr=None, src=''):
wx = lg.get_handle()
if date_str is None or len(date_str) ==0 :
date_str = datetime.now().strftime("%Y%m%d")
# wx.info("[RT_BL][baseline_big_deal] [{}--{}]大单基线设立[{}]".format(time_frame_arr[0], time_frame_arr[1],date_str))
begin_t_stamp = int(time.mktime(time.strptime(date_str+time_frame_arr[0], "%Y%m%d%H:%M")))
end_t_stamp = int(time.mktime(time.strptime(date_str+time_frame_arr[1], "%Y%m%d%H:%M")))
# 163 数据源，开始时间 后退5分钟
# if src == '163':
# begin_t_stamp -= 300
# time_frame_arr[0] = time.strftime("%H:%M", time.localtime(begin_t_stamp))
# 基线起止时间颠倒，互换
if begin_t_stamp > end_t_stamp:
begin_t_stamp, end_t_stamp = end_t_stamp, begin_t_stamp
baseline_big_deal_df = pd.DataFrame()
for id in rt.rt_dict_df.keys():
# 从RT 数据中筛选出 基线时间段 内的交易记录
rt_df = rt.rt_dict_df[id].loc[ (rt.rt_dict_df[id]['time_stamp'] >= begin_t_stamp) & ( rt.rt_dict_df[id]['time_stamp'] < end_t_stamp)].copy()
if rt_df is None or rt_df.empty:
wx.info("[RT_BL][baseline_big_deal] [{}] 在[{}-{}]期间交易数据为空，开始处理下一支股票".format(id, time_frame_arr[0], time_frame_arr[1]))
continue
# ID 的所有成交量
rt_df['amount'] = rt_df['vol'] * rt_df['price']
rt_amount = rt_df['amount'].sum()
rt_df['io_amount'] = rt_df['amount'] * rt_df['type']
# 内外盘、中性盘的数量统计、金额统计
rt_sell_qty = rt_df.loc[rt_df["type"] == -1].shape[0]
rt_buy_qty = rt_df.loc[rt_df["type"] == 1].shape[0]
rt_air_qty = rt_df.loc[rt_df["type"] == 0].shape[0]
rt_buy_amount = rt_df.loc[rt_df["type"] == 1].amount.sum()
rt_sell_amount = rt_df.loc[rt_df["type"] == -1].amount.sum()
rt_air_amount = rt_df.loc[rt_df["type"] == 0].amount.sum()
# 成交明细中的 大单列表
rt_big_df = rt_df.loc[rt_df['amount'] >= self.rt_big_amount,]
# 大单的数量
rt_big_qty = len(rt_big_df)
# 大单买入、卖出金额合计
rt_big_amount_sum_abs = rt_big_df['amount'].sum()
# 大单买入 卖出对冲后的金额
rt_big_amount_sum_io = rt_big_df['io_amount'].sum()
# 大单金额 占 总成交量的比例
big_abs_amount_pct = rt_big_amount_sum_abs/rt_amount
# 大单净买入 占 总成交量的比例
big_io_amount_pct = rt_big_amount_sum_io / rt_amount
# 平均每分钟的 大单买入、卖出金额
# rt_ave_big_amount_per_min_abs = rt_big_amount_sum_abs/((rt_end_time-rt_begin_time)/60)
# 卖盘的 大单金额
rt_big_sell_df = rt_big_df.loc[(rt_big_df['type'] < 0),]
rt_big_sell_amount = rt_big_sell_df['amount'].sum()
rt_big_sell_amount_pct = rt_big_sell_amount/rt_amount
# 买盘的 大单金额
rt_big_buy_df = rt_big_df.loc[(rt_big_df['type'] > 0),]
rt_big_buy_amount = rt_big_buy_df['amount'].sum()
rt_big_buy_amount_pct = rt_big_buy_amount/rt_amount
rt_baseline = {"id":id, "date":date_str,"t_frame":"-".join(time_frame_arr), "big_qty":rt_big_qty,
"big_abs_pct":big_abs_amount_pct, "big_io_pct":big_io_amount_pct,
"big_buy_pct":rt_big_buy_amount_pct, "big_sell_pct":rt_big_sell_amount_pct,
"amount":rt_amount, "sell_qty":rt_sell_qty, "sell_amount":rt_sell_amount,
"buy_qty":rt_buy_qty, "buy_amount":rt_buy_amount,
"air_qty":rt_air_qty, "air_amount":rt_air_amount}
if baseline_big_deal_df is None or baseline_big_deal_df.empty:
baseline_big_deal_df = pd.DataFrame([rt_baseline])
else:
baseline_big_deal_df = baseline_big_deal_df.append(pd.DataFrame([rt_baseline]))
if baseline_big_deal_df is None or baseline_big_deal_df.empty:
wx.info("[RT_BL][baseline_big_deal] [{}-{}] 大单基线交易数据为空，退出".format(time_frame_arr[0], time_frame_arr[1]))
return None
else:
cols = ['id','date','t_frame','big_qty','big_abs_pct','big_io_pct','big_buy_pct','big_sell_pct',
'amount','sell_qty','sell_amount','buy_qty','buy_amount','air_qty','air_amount']
baseline_big_deal_df = baseline_big_deal_df.loc[:,cols]
baseline_big_deal_df.fillna(0,inplace=True)
baseline_big_deal_df.reset_index(drop=True, inplace=True)
wx.info("[RT_BL][baseline_big_deal]大单数据基线[{}-{}]更新完毕".format(time_frame_arr[0], time_frame_arr[1]))
# wx.info("[RT_BL][baseline_big_deal] [{}-{}]数据基线更新完毕".format(time_frame_arr[0], time_frame_arr[1]))
return baseline_big_deal_df
def set_baseline_PA(self, rt=None, date_str='', time_frame_arr=None, src=''):
wx = lg.get_handle()
if date_str is None or len(date_str) == 0:
date_str = datetime.now().strftime("%Y%m%d")
# wx.info("[RT_BL][baseline_PA] [{}--{}]量价基线设立[{}]".format(time_frame_arr[0], time_frame_arr[1], date_str))
# 半个小时 时间段，来自 rebase_rt_data 设定
t_frame_begin_stamp = int(time.mktime(time.strptime(date_str + time_frame_arr[0], "%Y%m%d%H:%M")))
t_frame_end_stamp = int(time.mktime(time.strptime(date_str + time_frame_arr[1], "%Y%m%d%H:%M")))
# 基线起止时间颠倒，互换
if t_frame_begin_stamp > t_frame_end_stamp:
t_frame_begin_stamp, t_frame_end_stamp = t_frame_end_stamp, t_frame_begin_stamp
baseline_PA_df = pd.DataFrame()
# 使用 rt_PA_resample_agg_secs 设定的时间段，聚合resample 的数据
begin_t_stamp = t_frame_begin_stamp
if (t_frame_end_stamp - t_frame_begin_stamp) % self.rt_PA_resample_agg_secs == 0:
icount = (t_frame_end_stamp - t_frame_begin_stamp) / self.rt_PA_resample_agg_secs
else:
icount = (t_frame_end_stamp - t_frame_begin_stamp) // self.rt_PA_resample_agg_secs + 1
while icount > 0:
icount = icount - 1
end_t_stamp = begin_t_stamp + self.rt_PA_resample_agg_secs
if end_t_stamp > t_frame_end_stamp:
end_t_stamp = t_frame_end_stamp
agg_t_frame = [time.strftime("%H:%M", time.localtime(begin_t_stamp)),
time.strftime("%H:%M", time.localtime(end_t_stamp))]
for id in rt.rt_dict_df.keys():
# 从RT 数据中筛选出 rt_PA_resample_agg_secs 时间长度 （配置文件 10分钟）的交易记录
rt_df = rt.rt_dict_df[id].loc[(rt.rt_dict_df[id]['time_stamp'] >= begin_t_stamp) & (
rt.rt_dict_df[id]['time_stamp'] < end_t_stamp)].copy()
# 增加一列 pandas 的日期格式，用来做 rolling 或 resample 的index列
# rt_df['pd_time'] = date_str + " " + rt_df['time_str']
# rt_df['pd_time'] = pd.to_datetime(rt_df['pd_time'], format="%Y%m%d %H:%M:%S")
if rt_df is None or len(rt_df) == 0:
wx.info(
"[RT_BL][baseline_PA][{}] 在[{}-{}] 空交易数据，开始处理下一支股票".format(id, agg_t_frame[0], agg_t_frame[1]))
continue
rt_df = rt_df.sort_values(by="time_stamp", ascending=True)
rt_df['amount'] = rt_df['price'] * rt_df['vol']
# rt_df.set_index('pd_time', inplace=True)
rt_df['price'] = pd.to_numeric(rt_df['price'])
id_baseline_PA_amount = rt_df['amount'].sum()
id_baseline_PA_pct_chg = rt_df['price'].max() / rt_df['price'].min() - 1
id_baseline_PA_pct_chg_enhanced = id_baseline_PA_pct_chg * 500000000
# 量价向量长度
id_baseline_PA = pow(pow(id_baseline_PA_amount, 2) + pow(id_baseline_PA_pct_chg_enhanced, 2), 0.5)
# 量价向量方向, Time(High Price) - Time (Low Price) > 0 上涨； <0 下跌, （高价时间 - 低价时间）
if rt_df.price.idxmax() > rt_df.price.idxmin():
id_baseline_PA_pct_dir = 1
elif rt_df.price.idxmax() < rt_df.price.idxmin():
id_baseline_PA_pct_dir = -1
else:
id_baseline_PA_pct_dir = 0
# 量价向量角度, X轴 涨跌幅度 ， Y轴 成交金额
# 数值越小：小金额，大幅度涨跌
# 数值越大：大金额，小幅度涨跌
if id_baseline_PA_pct_chg_enhanced == 0:
id_baseline_PA_angle = 0
else:
id_baseline_PA_angle = id_baseline_PA_amount/ id_baseline_PA_pct_chg_enhanced
pa_baseline = {"id": id, "date": date_str, "t_frame": "-".join(agg_t_frame),
"sample_time": self.rt_PA_resample_secs,
"bl_pa": id_baseline_PA,
"bl_pa_angle": id_baseline_PA_angle,
"bl_pct": id_baseline_PA_pct_chg,
"bl_amount": id_baseline_PA_amount,
"bl_dir":id_baseline_PA_pct_dir
}
if baseline_PA_df is None or baseline_PA_df.empty:
baseline_PA_df = pd.DataFrame([pa_baseline])
else:
baseline_PA_df = baseline_PA_df.append(pd.DataFrame([pa_baseline]))
# while 循环，设定下一次的开始时间戳
begin_t_stamp = end_t_stamp
if end_t_stamp >= t_frame_end_stamp:
break
if baseline_PA_df is None or baseline_PA_df.empty:
wx.info("[RT_BL][baseline_PA] [{}-{}] 量价基线交易数据为空，退出".format(time_frame_arr[0], time_frame_arr[1]))
return None
else:
cols = ['id', 'date', 't_frame', 'sample_time',
'bl_pa', 'bl_pa_angle', 'bl_pct', 'bl_amount','bl_dir']
baseline_PA_df = baseline_PA_df.loc[:, cols]
baseline_PA_df.fillna(0, inplace=True)
baseline_PA_df.reset_index(drop=True, inplace=True)
wx.info("[RT_BL][baseline_PA] PA数据基线[{}-{}]更新完毕".format(agg_t_frame[0], agg_t_frame[1]))
# wx.info("[RT_BL][baseline_PA] [{}-{}] 量价数据基线更新完毕".format(time_frame_arr[0], time_frame_arr[1]))
return baseline_PA_df
def set_baseline_PA_old(self, rt=None, date_str='', time_frame_arr=None, src=''):
wx = lg.get_handle()
if date_str is None or len(date_str) == 0:
date_str = datetime.now().strftime("%Y%m%d")
wx.info("[RT_BL][baseline_PA] [{}][{}--{}]量价基线设立".format(date_str, time_frame_arr[0],time_frame_arr[1]))
# 半个小时 时间段，来自 rebase_rt_data 设定
t_frame_begin_stamp = int(time.mktime(time.strptime(date_str + time_frame_arr[0], "%Y%m%d%H:%M")))
t_frame_end_stamp = int(time.mktime(time.strptime(date_str + time_frame_arr[1], "%Y%m%d%H:%M")))
# 163 数据源，开始时间 后退5分钟
if src == '163':
t_frame_begin_stamp -= 300
time_frame_arr[0] = time.strftime("%H:%M", time.localtime(t_frame_begin_stamp))
# 基线起止时间颠倒，互换
if t_frame_begin_stamp > t_frame_end_stamp:
t_frame_begin_stamp, t_frame_end_stamp = t_frame_end_stamp, t_frame_begin_stamp
baseline_PA_df = pd.DataFrame()
# 使用 rt_PA_resample_agg_secs 设定的时间段，聚合resample 的数据
begin_t_stamp = t_frame_begin_stamp
if (t_frame_end_stamp - t_frame_begin_stamp) % self.rt_PA_resample_agg_secs == 0:
icount = (t_frame_end_stamp - t_frame_begin_stamp) / self.rt_PA_resample_agg_secs
else:
icount = (t_frame_end_stamp - t_frame_begin_stamp) // self.rt_PA_resample_agg_secs +1
while icount > 0:
icount = icount -1
end_t_stamp = begin_t_stamp + self.rt_PA_resample_agg_secs
if end_t_stamp > t_frame_end_stamp:
end_t_stamp = t_frame_end_stamp
agg_t_frame = [time.strftime("%H:%M", time.localtime(begin_t_stamp)),
time.strftime("%H:%M", time.localtime(end_t_stamp))]
for id in rt.rt_dict_df.keys():
id_baseline_PA_df = pd.DataFrame()
wx.info("[RT_BL][baseline_PA]开始更新[{}]的PA数据基线[{}-{}]".format(id, agg_t_frame[0], agg_t_frame[1]))
# 从RT 数据中筛选出 rt_PA_resample_agg_secs 时间长度 （配置文件 10分钟）的交易记录
rt_df = rt.rt_dict_df[id].loc[ (rt.rt_dict_df[id]['time_stamp'] >= begin_t_stamp) & ( rt.rt_dict_df[id]['time_stamp'] <= end_t_stamp)].copy()
# 增加一列 pandas 的日期格式，用来做 rolling 或 resample 的index列
rt_df['pd_time'] = date_str +" "+ rt_df['time_str']
rt_df['pd_time'] = pd.to_datetime(rt_df['pd_time'], format="%Y%m%d %H:%M:%S")
if rt_df is None or len(rt_df) == 0:
wx.info("[RT_BL][baseline_PA][{}] 在[{}-{}] 空交易数据，开始处理下一支股票".format(id, agg_t_frame[0], agg_t_frame[1]))
continue
rt_df = rt_df.sort_values(by="time_stamp", ascending=True)
rt_df['amount'] = rt_df['price']*rt_df['vol']
rt_df.set_index('pd_time', inplace=True)
rt_df['price'] = pd.to_numeric(rt_df['price'])
id_baseline_PA_df['amount'] = rt_df['amount'].sum()
id_baseline_PA_df['min_price'] = rt_df['price'].min()
id_baseline_PA_df['max_price'] = rt_df['price'].max()
id_baseline_PA_df['pct_chg'] = id_baseline_PA_df['max_price'] / id_baseline_PA_df['min_price'] - 1
id_baseline_PA_df['pct_chg_enhanced'] = (id_baseline_PA_df['max_price'] / id_baseline_PA_df['min_price'] - 1) * 500000000
# Time(High Price) - Time (Low Price) > 0 上涨； >0 下跌, （高价时间 - 低价时间）
id_baseline_PA_df['pct_up_down'] = rt_df.price.idxmax() - rt_df.price.idxmin()
# 将 pd.datetime64 之差 转换成 float 类型，方便判断 时间切片内的涨跌
id_baseline_PA_df['pct_up_down'] = pd.to_numeric(rt_df.price.idxmax() - rt_df.price.idxmin())
"""
id_baseline_PA_df['amount'] = rt_df['amount'].resample(self.rt_PA_resample_secs, label='left', closed='left' ).sum()
id_baseline_PA_df['min_price'] = rt_df['price'].resample(self.rt_PA_resample_secs, label='left', closed='left' ).min()
id_baseline_PA_df['max_price'] = rt_df['price'].resample(self.rt_PA_resample_secs, label='left', closed='left' ).max()
id_baseline_PA_df['pct_chg'] = id_baseline_PA_df['max_price'] / id_baseline_PA_df['min_price'] -1
id_baseline_PA_df['pct_chg_enhanced'] = (id_baseline_PA_df['max_price'] / id_baseline_PA_df['min_price'] -1 )*500000000
# Time(High Price) - Time (Low Price) > 0 上涨； >0 下跌, （高价时间 - 低价时间）
id_baseline_PA_df['pct_up_down'] = rt_df['price'].resample(self.rt_PA_resample_secs, label='left', closed='left' ).apply(lambda x: x.idxmax()- x.idxmin()
if len(x) > 0
else (pd.to_datetime(0)-pd.to_datetime(0)))
"""
id_baseline_PA_df.fillna(0, inplace=True)
# 過濾掉 成交量 == 0 & 價格變動 ==0 的 時間段記錄
id_baseline_PA_df = id_baseline_PA_df.loc[(id_baseline_PA_df['amount'] > 0)&(id_baseline_PA_df['pct_chg_enhanced'] >0),]
# 量价向量长度
id_baseline_PA_df['pa_vector'] = pow( pow(id_baseline_PA_df['amount'],2) + pow(id_baseline_PA_df['pct_chg_enhanced'],2),0.5)
# 量价向量方向
id_baseline_PA_df['pct_dir'] = id_baseline_PA_df['pct_up_down'].apply(lambda x: x/abs(x) if x != 0 else 0)
# 量价向量角度, X轴 涨跌幅度 ， Y轴 成交金额
# 数值越小：小金额，大幅度涨跌
# 数值越大：大金额，小幅度涨跌
id_baseline_PA_df['pa_angle'] = id_baseline_PA_df['amount'] / id_baseline_PA_df['pct_chg_enhanced']
# 上涨向量，去极值后，求均值 标准差
id_up_baseline_PA_df = id_baseline_PA_df.loc[id_baseline_PA_df['pct_up_down']>0,]
if id_up_baseline_PA_df is not None and len(id_up_baseline_PA_df) >0:
id_up_baseline_PA_df = self._pa_df_(id_up_baseline_PA_df, col='pa_vector') # 去极值函数（1倍标准差范围内的向量），获得计算均值的Dataframe
id_up_baseline_PA_df = self._pa_df_(id_up_baseline_PA_df, col='pa_angle') # 去极值函数（1倍标准差范围内的向量），获得计算均值的Dataframe
id_up_baseline_PA_df = self._pa_df_(id_up_baseline_PA_df, col='amount') # 去极值函数（1倍标准差范围内的向量），获得计算均值的Dataframe
id_up_baseline_PA_df = self._pa_df_(id_up_baseline_PA_df, col='pct_chg') # 去极值函数（1倍标准差范围内的向量），获得计算均值的Dataframe
id_up_bl_PA_ave = id_up_baseline_PA_df['pa_vector'].mean() # 向量长度均值
id_up_bl_pct_ave = id_up_baseline_PA_df['pct_chg'].mean() # 涨幅均值
id_up_bl_amount_ave = id_up_baseline_PA_df['amount'].mean() # 涨幅对应的成交金额均值
id_up_bl_PA_angle_ave = id_up_baseline_PA_df['pa_angle'].mean() # 向量角度均值
else:
id_up_bl_PA_ave = 0
id_up_bl_pct_ave = 0
id_up_bl_amount_ave = 0
id_up_bl_PA_angle_ave = 0
# 下跌向量，去极值后，求均值 标准差
id_down_baseline_PA_df = id_baseline_PA_df.loc[id_baseline_PA_df['pct_up_down']<0,]
if id_down_baseline_PA_df is not None and len(id_down_baseline_PA_df) >0:
id_down_baseline_PA_df = self._pa_df_(id_down_baseline_PA_df, col='pa_vector')
id_down_baseline_PA_df = self._pa_df_(id_down_baseline_PA_df, col='pa_angle')
id_down_baseline_PA_df = self._pa_df_(id_down_baseline_PA_df, col='amount')
id_down_baseline_PA_df = self._pa_df_(id_down_baseline_PA_df, col='pct_chg')
id_down_bl_PA_ave = id_down_baseline_PA_df['pa_vector'].mean()
id_down_bl_pct_ave = -1 * id_down_baseline_PA_df['pct_chg'].mean() # 跌幅均值，转换成负数
id_down_bl_amount_ave = id_down_baseline_PA_df['amount'].mean() # 跌幅对应的成交金额均值
id_down_bl_PA_angle_ave = -1 * id_down_baseline_PA_df['pa_angle'].mean() # 向量角度均值
else:
id_down_bl_PA_ave = 0
id_down_bl_pct_ave = 0
id_down_bl_amount_ave = 0
id_down_bl_PA_angle_ave = 0
pa_baseline = {"id":id, "date":date_str,"t_frame":"-".join(agg_t_frame), "sample_time":self.rt_PA_resample_secs,
"up_bl_pa_ave":id_up_bl_PA_ave,
"up_bl_pct_ave":id_up_bl_pct_ave , "up_bl_amount_ave":id_up_bl_amount_ave ,
"down_bl_pa_ave":id_down_bl_PA_ave,
"down_bl_pct_ave": id_down_bl_pct_ave, "down_bl_amount_ave": id_down_bl_amount_ave,
"up_bl_pa_angle_ave":id_up_bl_PA_angle_ave,
"down_bl_pa_angle_ave":id_down_bl_PA_angle_ave}
if baseline_PA_df is None or baseline_PA_df.empty:
baseline_PA_df = pd.DataFrame([pa_baseline])
else:
baseline_PA_df = baseline_PA_df.append(pd.DataFrame([pa_baseline]))
# while 循环，设定下一次的开始时间戳
begin_t_stamp = end_t_stamp
if end_t_stamp >= t_frame_end_stamp:
break
# 使用rolling 滑动窗口 取样，放弃这种方式
# id_baseline_PA_df['amount'] = rt_df['amount'].rolling('20s').sum()
# id_baseline_PA_df['max_price_index'] = pd.rolling_max(rt_df['price'], freq='20s')#.apply(self.__pct_up_down__, raw=False)
# id_baseline_PA_df['max_price_index'] = rt_df['price'].rolling_max('20s')#.apply(self.__pct_up_down__, raw=False)
# id_baseline_PA_df['min_price_index'] = rt_df['price'].rolling_min('20s')#.apply(_pct_up_down_, raw=False)
# id_baseline_PA_df['pct_up_down'] = id_baseline_PA_df.apply(self.__pct_up_down__)
if baseline_PA_df is None or baseline_PA_df.empty:
wx.info("[RT_BL][baseline_PA] [{}-{}] 量价基线交易数据为空，退出".format(time_frame_arr[0], time_frame_arr[1]))
return None
else:
cols = ['id', 'date', 't_frame', 'sample_time',
'up_bl_pa_ave', 'up_bl_pa_angle_ave', 'up_bl_pct_ave', 'up_bl_amount_ave',
'down_bl_pa_ave', 'down_bl_pa_angle_ave', 'down_bl_pct_ave', 'down_bl_amount_ave']
baseline_PA_df = baseline_PA_df.loc[:, cols]
baseline_PA_df.fillna(0, inplace=True)
baseline_PA_df.reset_index(drop=True, inplace=True)
#
# wx.info("[RT_BL][baseline_PA] 去除极值前记录数[{}]".format(len(baseline_PA_df)))
# baseline_PA_df =self._clr_extreme_data(pa_df=baseline_PA_df)
# wx.info("[RT_BL][baseline_PA] 去除极值后记录数[{}]".format(len(baseline_PA_df)))
wx.info("[RT_BL][baseline_PA] [{}-{}] 量价数据基线更新完毕".format(time_frame_arr[0], time_frame_arr[1]))
return baseline_PA_df
def trans_day(self):
wx = lg.get_handle()
today = date.today().strftime('%Y%m%d')
yesterday = (date.today() + timedelta(days=-1)).strftime('%Y%m%d')
wx.info("Yesterday:{} ---- Today date: {}".format(yesterday,today))
return self.api.trade_cal(exchange='', start_date=yesterday, end_date=today)
def __del__(self):
wx = lg.get_handle()
wx.info("db_ops : {}: __del__ called".format(self))
def json_parse(self, id=None, json_str=None):
wx = lg.get_handle()
if id is None:
wx.info("[RT_East][json_parse] 传入参数 股票ID 为空，退出")
return None
json_str = re.sub(r'jQuery\w+\(', r'', json_str)[:-2]
if json_str is not None:
json_obj = json.loads(json_str)
if json_obj is None:
wx.info("[RT_East][json_parse] JSON 对象为空，退出")
return None
# begin_time_str = json_obj['begin']
# end_time_str = json_obj['end']
# if len(json_obj['zhubi_list']) == 0:
# return None
time_str = jsonpath(json_obj, '$..data..data..t')
if time_str == False:
return [None, None]
price_str = jsonpath(json_obj, '$..data..data..p')
vol_str = jsonpath(json_obj, '$..data..data..v')
dir_str = jsonpath(json_obj, '$..data..data..bs')
rt_data = [dir_str, price_str, vol_str, time_str]
df = pd.DataFrame(rt_data)
df1 = df.T
df1.rename(columns={
0: 'type',
1: 'price',
2: 'vol',
3: 'time_str'
},
inplace=True)
# 删除集合竞价未成交的记录
df1 = df1[~(df1['type'].isin([4]))]
# -1 卖盘内盘 ； 1 卖票外盘， 与163 保持统计
df1['type'] = df1['type'].map({1: int(-1), 2: int(1)})
df1['time_str'] = df1['time_str'].astype('str')
df1['time_str'] = df1['time_str'].apply(
lambda x: ':'.join([x[:-4], x[-4:]]))
df1['time_str'] = df1['time_str'].apply(
lambda x: ':'.join([x[:-2], x[-2:]]))
df1['time_stamp'] = df1['time_str'].apply(lambda x: int(
time.mktime(time.strptime(self.date_str + x, '%Y%m%d%H:%M:%S'))))
df1['price'] = df1['price'] / 1000
df1['vol'] = df1['vol'] * 100
# date = [re.sub(r'-', '', tmp[0:10]) for tmp in dt]
# type = jsonpath(json_obj, '$..TRADE_TYPE')
# price = jsonpath(json_obj, '$..PRICE')
# vol = jsonpath(json_obj, '$..VOLUME_INC')
# time_stamp = jsonpath(json_obj, '$..DATE..sec')
# time_stamp_usec = jsonpath(json_obj, '$..DATE..usec')
# time_str = jsonpath(json_obj, '$..DATE_STR')
# rt_163_data = [seq, type, price, vol, time_stamp, time_str]
# rt_163_data = [seq, type, price, vol, time_stamp, time_stamp_usec, time_str]
# df = pd.DataFrame(rt_163_data)
ret_time_arr = [
time.strftime('%H:%M:%S', time.localtime(df1.time_stamp.min())),
time.strftime('%H:%M:%S', time.localtime(df1.time_stamp.max()))
]
if id in self.rt_dict_df.keys():
self.rt_dict_df[id] = self.rt_dict_df[id].append(
df1, sort=False).drop_duplicates()
self.rt_dict_df[id] = self.rt_dict_df[id].sort_values(
by="time_str", ascending=False)
else:
self.rt_dict_df[id] = df1
return ret_time_arr
def rebase_rt_data(rt=None, src='', date_str = ''):
wx = lg.get_handle()
# 股票代码数组，由 rt 对象内部变量 带入
if rt.id_arr is None:
wx.info("[Rebase_RT_Data] 股票列表为空，退出")
return None
# 股票代码数组，由 rt 对象内部变量 带入
if date_str is None or len(date_str) == 0:
date_str = (datetime.today()).strftime('%Y%m%d')
wx.info("[Rebase_RT_Data] 未指定回溯的日期，默认使用 {}".format(date_str))
# 起始时间，作为查询实时交易数据的时间节点
# begin_time_arr= ['13:00','09:25','09:30','09:40','09:50']#,'10:00','10:30','11:00','13:00','13:30','14:00','14:30','14:40','14:50']#
begin_time_arr= ['09:25','09:30','09:40','09:50','10:00','10:30','11:00','13:00','13:30','14:00','14:30','14:40','14:50']#
end_time_arr = ['09:30','09:40','09:50','10:00','10:30','11:00','11:30','13:30','14:00','14:30','14:40','14:50','15:00']#
# end_time_arr = ['13:30','09:30','09:40','09:50','10:00']#,'10:30','11:00','11:30','13:30','14:00','14:30','14:40','14:50','15:00']#
# 保持全部的 baseline 数据，去极值后，一次性导入数据库
final_bl_big_deal_df = pd.DataFrame()
# 保持全部的 baseline 数据，一次性导入数据库
final_bl_pa_df = pd.DataFrame()
bl = rt_bl()
for index in range(len(begin_time_arr)):
time_inc = 5
begin_time_stamp = int(time.mktime(time.strptime(date_str+begin_time_arr[index], "%Y%m%d%H:%M")))
end_time_stamp = int(time.mktime(time.strptime(date_str+end_time_arr[index], "%Y%m%d%H:%M")))
while begin_time_stamp < end_time_stamp:
# rt 对象在主函数生成，传入此函数，添加
if src == '163':
time_str = time.strftime("%H:%M:%S", time.localtime(begin_time_stamp+300)) # 163 的5分钟偏移量
wx.info("[Rebase_RT_Data] 从[{}] 获得[{}] 支股票的交易数据 [{}]-[{}] ".format(src, len(rt.id_arr), date_str, time_str ))
else:
time_str = time.strftime("%H:%M:%S", time.localtime(begin_time_stamp))
for icount, id in enumerate(rt.id_arr):
if src == '163': # 每支股票按 5分钟时间 递增查询，直到 时间递增后超过 end_time_arr[index]
json_str = rt.get_json_str(id=id, time_str=time_str)
time_range = rt.json_parse(id=id, json_str=json_str)
if time_range is None:
wx.info("[Rebase_RT_Data][{}/{}] [{}] [{}-{}] 交易数据不存在".
format(icount + 1, len(rt.id_arr), id,date_str, time_str))
else:
wx.info("[Rebase_RT_Data][{}/{}] {} [{}--{}]逐笔交易数据[{}-{}]".
format(icount + 1, len(rt.id_arr), id, time_range[0], time_range[1], date_str, time_str))
time.sleep(0.5)
elif src == 'east':
# 每支股票 按 begin_time_arr[index] - end_time_arr[index] 查询，存入 RT对象
# 完成所有股票后，进入下一个时间段查询
wx.info("[Rebase_RT_Data][{}/{}] ==================> {} 开始获取目标时间段[{}---{}-{}]".
format(icount + 1, len(rt.id_arr), id, date_str, begin_time_arr[index], end_time_arr[index]))
rt.get_json_str(id=id, time_str = begin_time_arr[index] +"-"+end_time_arr[index])
# 下一次循环的 起始时间
if src == '163':
begin_time_stamp += time_inc*60
elif src == 'east':
break # 已完成所有股票，在本时间段的查询，直接进入下一个时间段查询
# 大单交易的 基线数据，每个时间片【begin_time_arr，end_time_arr】 每支股票产生一条记录
baseline_big_deal_df = bl.set_baseline_big_deal(rt=rt, date_str=date_str, time_frame_arr=[begin_time_arr[index], end_time_arr[index]], src= src)
if final_bl_big_deal_df is None or len(final_bl_big_deal_df) == 0:
final_bl_big_deal_df = baseline_big_deal_df
else:
final_bl_big_deal_df = final_bl_big_deal_df.append(baseline_big_deal_df)
# 量价 基线数据，每个时间片【begin_time_arr，end_time_arr】 每支股票产生一条记录
baseline_PA_df = bl.set_baseline_PA(rt=rt, date_str=date_str, time_frame_arr=[begin_time_arr[index], end_time_arr[index]], src= src)
if final_bl_pa_df is None or len(final_bl_pa_df) == 0:
final_bl_pa_df = baseline_PA_df
else:
final_bl_pa_df = final_bl_pa_df.append(baseline_PA_df)
if src == '163':
# 释放 RT 对象的内部变量，只保留 最后15分钟的交易数据
rt.clr_rt_data(minutes=35)
elif src == 'east':
rt.clr_rt_data(stamp = end_time_stamp)
# for循环结束，进入下一个时间段（半小时）
# 全天交易时间结束，先做累加
# cols = ['id', 'date', 't_frame', 'big_qty', 'big_abs_pct', 'big_io_pct', 'big_buy_pct', 'big_sell_pct',
# 'amount', 'sell_qty', 'sell_amount', 'buy_qty', 'buy_amount', 'air_qty', 'air_amount']
final_cu_bl_big_deal_df = pd.DataFrame()
for each_id in final_bl_big_deal_df.groupby(final_bl_big_deal_df['id']):
df_each_id = each_id[1].sort_values(by="t_frame", ascending=True)
df_each_id['cu_big_qty'] = df_each_id['big_qty'].cumsum()
df_each_id['cu_amount'] = df_each_id['amount'].cumsum()
df_each_id['cu_sell_qty'] = df_each_id['sell_qty'].cumsum()
df_each_id['cu_sell_amount'] = df_each_id['sell_amount'].cumsum()
df_each_id['cu_buy_qty'] = df_each_id['buy_qty'].cumsum()
df_each_id['cu_buy_amount'] = df_each_id['buy_amount'].cumsum()
df_each_id['cu_air_qty'] = df_each_id['air_qty'].cumsum()
df_each_id['cu_air_amount'] = df_each_id['air_amount'].cumsum()
if final_cu_bl_big_deal_df is None or len(final_cu_bl_big_deal_df) == 0:
final_cu_bl_big_deal_df = df_each_id
else:
final_cu_bl_big_deal_df = final_cu_bl_big_deal_df.append(df_each_id)
cols = ['id', 'date', 't_frame', 'big_qty', 'big_abs_pct', 'big_io_pct', 'big_buy_pct', 'big_sell_pct',
'amount', 'sell_qty', 'sell_amount', 'buy_qty', 'buy_amount', 'air_qty', 'air_amount',
'cu_big_qty','cu_amount','cu_sell_qty','cu_sell_amount','cu_buy_qty','cu_buy_amount',
'cu_air_qty','cu_air_amount']
final_cu_bl_big_deal_df = final_cu_bl_big_deal_df.loc[:, cols]
final_cu_bl_big_deal_df.fillna(0, inplace=True)
final_cu_bl_big_deal_df.reset_index(drop=True, inplace=True)
# 将基线数据导入数据库
bl.db_load_baseline_big_deal(df=final_cu_bl_big_deal_df)
# 再次对全天的 PA 数据去极值
# final_bl_pa_df = bl._clr_extreme_data(pa_df=final_bl_pa_df)
bl.db_load_baseline_PA(df=final_bl_pa_df)
def __del__(self):
wx = lg.get_handle()
wx.info("[OBJ] filter_fix : __del__ called")
def get_rt_data(rt=None, src='', date_str=''):
wx = lg.get_handle()
# 股票代码数组，由 rt 对象内部变量 带入
if rt.id_arr is None:
wx.info("[Get_RT_Data]: 股票列表为空，退出")
return None
if date_str is None or len(date_str) == 0:
date_str = (datetime.today()).strftime('%Y%m%d')
wx.info("[Get_RT_Data] 未指定交易日期，默认使用 {}".format(date_str))
my_timer = wx_timer(date_str=date_str)
# 判断 今日期 是否交易日
if my_timer.is_trading_date(date_str=date_str):
wx.info("{}是交易日，继续运行".format(date_str))
else:
wx.info("[Get_RT_Data]:{}不是交易日，退出实时交易数据获取 ".format(date_str))
return None
time_inc = 5 # 5分钟增量
# 从记录文件读取上一次查询实时交易的截止时间，本次查询的开始时间
# 获得的时间戳有两种情况 1）取整（半小时）的时间戳 2）空文件，自动处理成 09:25
# 记录文件名：日期_数据来源
begin_time_str = rt._get_last_record_()
if src == '163':
if begin_time_str is None :
begin_time_str = '09:25'
elif begin_time_str == '15:00':
wx.info("[Get_RT_Data] 今日文件记录已查询过所有实时交易，退出")
return False
elif src == 'east':
if begin_time_str is None :
begin_time_str = '09:25'
else: # 解析文件记录
record_dict = json.loads(begin_time_str)
begin_time_str = record_dict['time_str']
if begin_time_str == '15:00':
wx.info("[Get_RT_Data] 今日文件记录已查询过所有实时交易，退出")
return False
rt.record_page_dict = record_dict['page']
# 开始 获取实时交易的时间起点，并判断时间是否在交易时间
begin_time_stamp = int(time.mktime(time.strptime(date_str+begin_time_str[:5], "%Y%m%d%H:%M")))
# 返回值，ana_rt_data 的起点时间
ret_begin_time_stamp = begin_time_stamp
ret_zone = my_timer.tell_time_zone(t_stamp = begin_time_stamp)
if ret_zone[0] < 0:
begin_time_stamp = ret_zone[1]
# 当前时间，如果超出交易时间，则拉回到 交易时间
end_time_stamp = int(time.time())
end_time_str = time.strftime('%H:%M', time.localtime(time.time()))
ret_zone = my_timer.tell_time_zone(t_stamp = end_time_stamp)
record_stamp = ret_zone[2]
record_str = ''
if ret_zone[0] < 0:
end_time_stamp = ret_zone[1]
# if begin_time_stamp > end_time_stamp:
# wx.info("[Get_RT_Data] 查询间隔不足5分钟，需等待{}秒".format(begin_time_stamp-end_time_stamp))
# time.sleep(begin_time_stamp-end_time_stamp)
while begin_time_stamp < end_time_stamp:
# rt 对象在主函数生成，传入此函数，添加
if src == '163': # 从163 获取数据，时间偏移5分钟
time_str = time.strftime("%H:%M:%S", time.localtime(begin_time_stamp+300))
else:
time_str = time.strftime("%H:%M:%S", time.localtime(begin_time_stamp))
wx.info("[Get_RT_Data] 从[{}] 查询 [{}]支股票的 交易数据 [{}] ".format(src, len(rt.id_arr), time_str))
for icount, id in enumerate(rt.id_arr):
if src == '163':
# wx.info("[Get_RT_Data][{}:{}] {} 获取逐笔交易数据[{}]".format(icount+1,len(rt.id_arr),id, time_str))
json_str = rt.get_json_str(id=id, time_str=time_str)
# wx.info("[Get_RT_Data][{}:{}] {} 解析逐笔交易数据[{}]".format(icount+1,len(rt.id_arr),id, time_str))
time_range = rt.json_parse(id=id, json_str=json_str)
if time_range is None:
wx.info("[Get_RT_Data][{}/{}] {} [{}]逐笔交易数据 为空".format(icount + 1, len(rt.id_arr), id, time_str))
else:
wx.info("[Get_RT_Data][{}/{}] {} [{}--{}]逐笔交易数据[{}]".format(icount + 1, len(rt.id_arr), id, time_range[0],
time_range[1], time_str))
elif src == 'east':
# 每支股票 按 begin_time_arr[index] - end_time_arr[index] 查询，存入 RT对象
# 完成所有股票后，进入下一个时间段查询
wx.info("[Get_RT_Data][{}/{}] {} 开始获取目标时间段[{}---{}-{}]".
format(icount + 1, len(rt.id_arr), id, date_str, begin_time_str, end_time_str))
# 临时加入的ID，设置初设页面为0
if id not in rt.record_page_dict.keys():
rt.record_page_dict[id] =0
rt.get_json_str(id=id, time_str=begin_time_str + "-" + end_time_str, page_num = rt.record_page_dict[id])
if src == '163':
# 计算下一个循环的 起始时间， 和 文件记录时间，并调整
begin_time_stamp += time_inc*60
ret_zone = my_timer.tell_time_zone(t_stamp=begin_time_stamp)
record_stamp = ret_zone[2]
if ret_zone[0] == -3:
begin_time_stamp = ret_zone[1]
# begin_time_stamp == end_time_stamp 再进行一次循环
# begin_time_stamp > end_time_stamp 且差值 在 time_inc * 60 秒内，设置 begin == end
if begin_time_stamp >= end_time_stamp and begin_time_stamp-end_time_stamp < time_inc*60:
begin_time_stamp = end_time_stamp
elif src == 'east':
record_dict={'time_str':time.strftime('%H:%M', time.localtime(record_stamp)),
'page': rt.record_page_dict}
record_str = json.dumps(record_dict)
break
# 文件记录最近一次的实时交易数据时间
if src == '163':
time_str = time.strftime("%H:%M", time.localtime(record_stamp))
rt.f_record.write('\n'+time_str)
elif src == 'east':
if len(record_str) != 0:
rt.f_record.write('\n'+record_str)
rt.f_record.flush()
return ret_begin_time_stamp
class db_ops:
wx = lg.get_handle()
def __init__(self):
wx = lg.get_handle()
try:
self.h_conf = conf_handler(conf="rt_analyer.conf")
host = self.h_conf.rd_opt('db', 'host')
database = self.h_conf.rd_opt('db', 'database')
user = self.h_conf.rd_opt('db', 'user')
pwd = self.h_conf.rd_opt('db', 'pwd')
if pwd is None:
wx.info("[Err DB_OP]===> {0}:{1}:{2} need password ".format(
host, db, user))
raise Exception("Password is Null")
else:
# self.pwd = pwd
self.config = {
'host': host,
'user': user,
'password': pwd,
'database': database,
'charset': 'utf8',
'port': 3306 # 注意端口为int 而不是str
}
# self.handle = pymysql.connect(self.host, self.user, self.pwd, self.db_name)
self.handle = pymysql.connect(**self.config)
self.cursor = self.handle.cursor()
# wx.info("[OBJ] db_ops : __init__ called")
except Exception as e:
wx.info("Err occured in DB_OP __init__{}".format(e))
raise e
def __del__(self):
# wx = lg.get_handle()
# wx.info("[OBJ] db_ops : __del__ called")
pass
def get_trade_date(self, back_days=1):
self.rt_conf = conf_handler(conf="rt_analyer.conf")
tname = self.rt_conf.rd_opt('db', 'daily_table_cq_60')
sql = "select distinct date from " + tname + " order by date desc limit " + str(
back_days)
df = self._exec_sql(sql=sql)
df.sort_values(by='date', ascending=False, inplace=True)
# 取得最后一天的日期
trade_date = df.iloc[-1][0]
return trade_date
def select_table(self, t_name=None, where="", order="", limit=100):
wx = lg.get_handle()
if t_name is not None:
sql = "select * from " + t_name + " " + where + " " + order + " limit " + str(
limit)
wx.info("[select_table] {}".format(sql))
df_ret = self._exec_sql(sql)
return df_ret
def _exec_sql(self, sql=None):
wx = lg.get_handle()
if sql is None:
return None
iCount = self.cursor.execute(sql)
self.handle.commit()
if iCount > 0:
# wx.info("[calc days vol] acquire {} rows of result".format(iCount))
arr_ret = self.cursor.fetchall()
if len(arr_ret) == 0:
wx.info("[_exec_sql] Empty Dataframe Returned : SQL {}".format(
sql))
return None
columnDes = self.cursor.description # 获取连接对象的描述信息
columnNames = [columnDes[i][0] for i in range(len(columnDes))]
df_ret = pd.DataFrame([list(i) for i in arr_ret],
columns=columnNames)
return df_ret
else:
wx.info(
"[_exec_sql] Empty Dataframe Returned : SQL {}".format(sql))
return None
def db_load_into_RT_BL_Big_Deal(self, df=None):
wx = lg.get_handle()
t_name = self.h_conf.rd_opt('db', 'rt_baseline_big')
if df is None:
wx.info("[db_load_into_RT_BL_Big_Deal]Err: dataframe is Empty,")
return -1
df_array = df.values.tolist()
i = 0
while i < len(df_array):
df_array[i] = tuple(df_array[i])
i += 1
# cols = ['id', 'date', 't_frame', 'big_qty', 'big_abs_pct', 'big_io_pct', 'big_buy_pct', 'big_sell_pct',
# 'amount', 'sell_qty', 'sell_amount', 'buy_qty', 'buy_amount', 'air_qty', 'air_amount',
# 'cu_big_qty', 'cu_amount', 'cu_sell_qty', 'cu_sell_amount', 'cu_buy_qty', 'cu_buy_amount',
# 'cu_air_qty', 'cu_air_amount']
sql = "REPLACE INTO "+t_name+" SET id=%s, date=%s, t_frame=%s, big_qty=%s, big_abs_pct=%s, big_io_pct=%s, " \
"big_buy_pct=%s, big_sell_pct=%s, amount=%s, sell_qty=%s, sell_amount=%s, buy_qty=%s, buy_amount=%s," \
"air_qty=%s, air_amount=%s, cu_big_qty=%s, cu_amount=%s, cu_sell_qty=%s, cu_sell_amount=%s, " \
"cu_buy_qty=%s, cu_buy_amount=%s, cu_air_qty=%s, cu_air_amount=%s"
self.cursor.executemany(sql, df_array)
self.handle.commit()
# 实时交易数据 监测的超阀值记录 导入数据库
def db_load_RT_MSG(self, df=None):
wx = lg.get_handle()
t_name = self.h_conf.rd_opt('db', 'rt_message')
if df is None or df.empty:
wx.info("[db_load_RT_MSG] 实时信息 DataFrame 为空，退出")
return
df_array = df.values.tolist()
i = 0
while i < len(df_array):
df_array[i] = tuple(df_array[i])
i += 1
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, t_frame=%s, type=%s, msg=%s"
self.cursor.executemany(sql, df_array)
self.handle.commit()
def db_load_into_RT_BL_PA(self, df=None):
wx = lg.get_handle()
t_name = self.h_conf.rd_opt('db', 'rt_baseline_PA')
if df is None:
wx.info("[db_load_into_RT_BL_PA]Err: dataframe is Empty,")
return -1
df_array = df.values.tolist()
i = 0
while i < len(df_array):
df_array[i] = tuple(df_array[i])
i += 1
# cols = ['id', 'date', 't_frame', 'sample_time',
# 'bl_pa', 'bl_pa_angle', 'bl_pct', 'bl_amount', 'bl_dir']
sql = "REPLACE INTO "+t_name+" SET id=%s, date=%s, t_frame=%s, sample_time=%s, " \
"bl_pa=%s, bl_pa_ang=%s, " \
"bl_pct=%s, bl_amount=%s, bl_dir=%s"
self.cursor.executemany(sql, df_array)
self.handle.commit()
def get_bl_pa(self, days=1):
t_name = self.h_conf.rd_opt('db', 'rt_baseline_PA')
sql = "select distinct date from " + t_name + " order by date desc limit " + str(
days)
date_df = self._exec_sql(sql=sql)
data_arr = date_df.date.values.tolist()
date_str = ",".join(data_arr)
sql = "select * from " + t_name + " where date in (" + date_str + ")"
ret_df = self._exec_sql(sql=sql)
return ret_df
def get_bl_big_deal(self, days=3):
wx = lg.get_handle()
t_name = self.h_conf.rd_opt('db', 'rt_baseline_big')
sql = "select distinct date from " + t_name + " order by date desc limit " + str(
days)
date_df = self._exec_sql(sql=sql)
data_arr = date_df.date.values.tolist()
date_str = ",".join(data_arr)
sql = "select * from " + t_name + " where date in (" + date_str + ")"
ret_df = self._exec_sql(sql=sql)
return ret_df
def get_cu_big_deal_date(self, date_str='', t_frame=''):
if date_str is None or len(date_str) == 0:
date_str = (date.today()).strftime('%Y%m%d')
t_name = self.h_conf.rd_opt('db', 'rt_baseline_big')
sql = "select id, sum(big_qty) as cu_big_qty, sum(amount) as cu_amount, " \
"sum(buy_qty) as cu_buy_qty, sum(buy_amount) as cu_buy_amount , " \
"sum(sell_qty) as cu_sell_qty, sum(sell_amount) as cu_sell_amount " \
"from "+ t_name+" where date= "+date_str+" and t_frame < '"+t_frame+"' group by id"
ret_df = self._exec_sql(sql=sql)
return ret_df
def db_load_into_NOTICE(self, df=None):
wx = lg.get_handle()
t_name = self.h_conf.rd_opt('db', 'notice_table')
if df is None or df.empty:
wx.info("[db_load_into_NOTICE] 公告信息为空，退出")
return
df['title'] = df['title'].apply(lambda x: x[:199])
df_array = df.values.tolist()
i = 0
while i < len(df_array):
df_array[i] = tuple(df_array[i])
i += 1
sql = "REPLACE INTO " + t_name + " SET ann_time=%s, id=%s, title=%s"
self.cursor.executemany(sql, df_array)
self.handle.commit()
def _filter_strength(self, df_side_left=None, df_side_right=None):
wx = lg.get_handle()
if df_side_left is None:
wx.info("[Filter Fix] filter_Left_Side DataFrame is Empty")
return None
if df_side_right is None:
wx.info("[Filter Fix] filter_Right_Side DataFrame is Empty")
return None
# 按日期排序，由小到大
df_side_right = df_side_right.sort_values('date',
ascending=False).copy()
# 涨幅 变整数，便于统计数量
df_side_left['pct_chg_int'] = df_side_left['pct_chg'].astype(
int).copy()
df_side_left.loc[df_side_left[(df_side_left['pct_chg'] >= 11)].index,
['pct_chg_int']] = 10
df_side_left.loc[df_side_left[(df_side_left['pct_chg'] <= -11)].index,
['pct_chg_int']] = -10
df_side_right['pct_chg_int'] = df_side_right['pct_chg'].astype(
int).copy()
df_side_right.loc[df_side_right[(
df_side_right['pct_chg'] >= 11)].index, ['pct_chg_int']] = 10
df_side_right.loc[df_side_right[(
df_side_right['pct_chg'] <= -11)].index, ['pct_chg_int']] = -10
# 读取 高点左侧 涨幅的权重表
high_left_power_conf = dict(
self.f_conf.rd_sec(sec='filter_high_left_power_table'))
zero_power_table = dict(zip(range(-10, 11),
[0] * 21)) # 从 -10% 到 10% 全部权重为0
high_left_power = 0
for key in high_left_power_conf.keys():
power_item = high_left_power_conf[key].split("#")
k = [int(x) for x in power_item[0].split(",")]
v = [int(x) for x in power_item[1].split(",")]
real_power_table = zero_power_table.copy()
real_power_table.update(dict(zip(k, v))) # 从配置文件读取权重，赋值
high_left_power_conf[key] = real_power_table.copy()
# Key 是天数，作为统计周期，获得这个周期内的交易数据
df_lastest_side = df_side_left.head(int(key))
pct_count = dict(df_lastest_side['pct_chg_int'].value_counts())
# 获得这个周期内的 high_left_power 得分
for key in pct_count.keys():
high_left_power += pct_count[key] * real_power_table[key]
# 读取 最新交易日 左侧 涨幅的权重表
cur_left_power_conf = dict(
self.f_conf.rd_sec(sec='filter_cur_left_power_table'))
zero_power_table = dict(zip(range(-10, 11),
[0] * 21)) # 从 -10% 到 10% 全部权重为0
cur_left_power = 0
for key in cur_left_power_conf.keys():
power_item = cur_left_power_conf[key].split("#")
k = [int(x) for x in power_item[0].split(",")]
v = [int(x) for x in power_item[1].split(",")]
real_power_table = zero_power_table.copy()
real_power_table.update(dict(zip(k, v))) # 从配置文件读取权重，赋值
cur_left_power_conf[key] = real_power_table.copy()
# Key 是天数，作为统计周期，获得这个周期内的交易数据
df_lastest_side = df_side_right.head(int(key))
pct_count = dict(df_lastest_side['pct_chg_int'].value_counts())
# 获得这个周期内的 left_power 得分
for key in pct_count.keys():
cur_left_power += pct_count[key] * real_power_table[key]
# 读取 高点右侧 涨幅的权重表
high_right_power_conf = dict(
self.f_conf.rd_sec(sec='filter_high_right_power_table'))
zero_power_table = dict(zip(range(-10, 11),
[0] * 21)) # 从 -10% 到 10% 全部权重为0
high_right_power = 0
for key in high_right_power_conf.keys():
power_item = high_right_power_conf[key].split("#")
k = [int(x) for x in power_item[0].split(",")]
v = [int(x) for x in power_item[1].split(",")]
real_power_table = zero_power_table.copy()
real_power_table.update(dict(zip(k, v))) # 从配置文件读取权重，赋值
high_right_power_conf[key] = real_power_table.copy()
# Key 是天数，作为统计周期，获得这个周期内的交易数据
df_lastest_side = df_side_right.tail(int(key))
pct_count = dict(df_lastest_side['pct_chg_int'].value_counts())
# 获得这个周期内的 left_power 得分
for key in pct_count.keys():
high_right_power += pct_count[key] * real_power_table[key]
ret_arr_power = [high_left_power, high_right_power, cur_left_power]
return ret_arr_power
def db_load_into_ind_xxx(self,
ind_type='ma',
ind_df=None,
stock_type=None,
data_src='cq'):
wx = lg.get_handle()
if stock_type is None:
wx.info(
"[db_load_into_ind_xxx] Err: {} Stock Type is Empty,".format(
ind_type))
return -1
if ind_df is None or ind_df.empty:
wx.info(
"[db_load_into_ind_xxx] Err: {} {} Data Frame is None or Empty"
.format(ind_type, stock_type))
return -1
ind_arry = ind_df.values.tolist()
i = 0
while i < len(ind_arry):
ind_arry[i] = tuple(ind_arry[i])
i += 1
tname_00 = self.h_conf.rd_opt('db',
ind_type + '_' + data_src + '_table_00')
tname_30 = self.h_conf.rd_opt('db',
ind_type + '_' + data_src + '_table_30')
tname_60 = self.h_conf.rd_opt('db',
ind_type + '_' + data_src + '_table_60')
tname_002 = self.h_conf.rd_opt(
'db', ind_type + '_' + data_src + '_table_002')
tname_68 = self.h_conf.rd_opt('db',
ind_type + '_' + data_src + '_table_68')
if re.match('002', stock_type) is not None:
t_name = tname_002
elif re.match('00', stock_type) is not None:
t_name = tname_00
elif re.match('30', stock_type) is not None:
t_name = tname_30
elif re.match('60', stock_type) is not None:
t_name = tname_60
elif re.match('68', stock_type) is not None:
t_name = tname_68
else:
wx.info(
"[db_load_into_ind_xxx] stock_type does NOT match ('002','00','30','60')"
)
if (ind_type == 'ma'):
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, ma_5=%s, ma_10=%s, ma_20=%s, ma_60=%s, " \
"ma_13=%s, ma_34=%s, ma_55=%s, ema_12=%s, ema_26=%s, DIF=%s, DEA=%s, " \
"bolling_mid=%s, bolling_top=%s, bolling_bottom=%s"
elif (ind_type == 'psy'):
sql = "REPLACE INTO " + t_name + " SET id=%s, date=%s, psy=%s"
else:
return None
i_scale = 1000
for i in range(0, len(ind_arry), i_scale):
tmp_arry = ind_arry[i:i + i_scale]
wx.info(
"[db_load_into_ind_xxx][{}] Loaded {} ~ {} , total {} ".format(
t_name, i, i + i_scale, len(ind_arry)))
self.db.cursor.executemany(sql, tmp_arry)
self.db.handle.commit()
def filter_side(self):
wx = lg.get_handle()
# if self.data_src == 'bt_qfq':
# start_date = self.f_start_date
# elif self.data_src == 'cq' or self.data_src == 'qfq':
# start_date = (date.today() + timedelta(days=-180)).strftime('%Y%m%d')
# else:
# wx.info("[Filter_fix] [filter_side] data src [{}], Err. ".format(self.data_src))
# return
if self.data_src == 'qfq':
tname_arr = [
self.daily_qfq_t_002, self.daily_qfq_t_68, self.daily_qfq_t_00,
self.daily_qfq_t_30, self.daily_qfq_t_60
]
elif self.data_src == 'bt_qfq':
tname_arr = [
self.bt_daily_qfq_t_00, self.bt_daily_qfq_t_30,
self.bt_daily_qfq_t_60, self.bt_daily_qfq_t_002,
self.bt_daily_qfq_t_68
]
elif self.data_src == 'cq':
tname_arr = [
self.daily_cq_t_00, self.daily_cq_t_30, self.daily_cq_t_60,
self.daily_cq_t_002, self.daily_cq_t_68
]
arr_filter_side = []
for t_name in tname_arr:
sql = "select id, date, high, low, close, 100*(close-pre_close)/pre_close as pct_chg from " + t_name + \
" where date > " + self.f_start_date
df_all_tmp = self.db._exec_sql(sql=sql)
if df_all_tmp is None or df_all_tmp.empty:
wx.info("[filter_side] {} 数据表未检出任何记录，继续处理...".format(t_name))
continue
df_all = df_all_tmp[-df_all_tmp.high.isin([0])]
# 按股票ID分组，再按high排序，取每组的第一条记录，即该股票ID的最高价记录
df_high = df_all.sort_values('high', ascending=False).groupby(
'id', as_index=False).first()
wx.info(
"[Filter_Curve] Completed Located the Highest Price Point in {}"
.format(t_name))
# df_left_side = df_all.groupby(['id']).apply(lambda x: self._find_left(x, df_high))
df_groupby_id = df_all.groupby(['id'])
for count, df_each_stock in enumerate(df_groupby_id):
# 获取最高点 左侧 \ 右侧 的数据记录
df_tmp_left = self._acquire_sides(id=df_each_stock[0],
x=df_each_stock[1].copy(),
df_high=df_high,
dir="left")
df_tmp_right = self._acquire_sides(id=df_each_stock[0],
x=df_each_stock[1].copy(),
df_high=df_high,
dir="right")
# 判断 左侧涨幅 是否满足 filter_growth_below_pct 要求
# 判断 右侧是否 创新低，条件低于左侧 3%
ret_LR = self._filter_LR(df_side_left=df_tmp_left,
df_side_right=df_tmp_right)
if ret_LR == 1:
wx.info(
"[Filter_Curve LR] {} / {} Filter {} LR ... [LR FOUND!!!] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
elif ret_LR == 0: # 左侧涨幅不达标
wx.info(
"[Filter_Curve LR] {} / {} Filter {} Left Raise Low... [LR PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
elif ret_LR == -1: # 右侧创新低
wx.info(
"[Filter_Curve LR] {} / {} Filter {} Right New Lowest ... [LR PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
# 计算该股票 收盘价 接近的最小黄金分割比例
min_golden_pct = self._filter_golden_price(
df_right_side=df_tmp_right)
if min_golden_pct > self.filter_golden_pct_request:
wx.info(
"[Filter_Curve Golden] {} / {} Filter {} ........ [Golden PCT FOUND!!!] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
else:
wx.info(
"[Filter_Curve Golden] {} / {} Filter {} ........ [Golden PCT PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
# 根据规则过滤 高点左侧\右侧 、
arr_power = self._filter_strength(df_side_left=df_tmp_left,
df_side_right=df_tmp_right)
# wx.info("{}:{}".format(df_each_stock[0],arr_power))
# 配置文件中所有周期的 Power 得分累计
if arr_power[0] < int(self.filter_high_left_power_request):
wx.info(
"[Filter_Curve Power] {} / {} Filter {} High Left Side Power... [PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
elif arr_power[1] < int(self.filter_high_right_power_request):
wx.info(
"[Filter_Curve Power] {} / {} Filter {} High Right Side Power... [PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
elif arr_power[2] < int(self.filter_cur_left_power_request):
wx.info(
"[Filter_Curve Power] {} / {} Filter {} Cur Left Side Power... [PASS] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
continue
else:
wx.info(
"[Filter_Curve Power] {} / {} Filter {} Power........ [FOUND!] "
.format(count + 1, len(df_groupby_id),
df_each_stock[0]))
arr_power.insert(0, df_each_stock[0])
arr_power.append(min_golden_pct)
arr_filter_side.append(arr_power)
df_filter_side = pd.DataFrame(
arr_filter_side,
columns=['股票代码', '高点左侧得分', '高点右侧得分', '今日左侧得分', '收盘价低于(高低点黄金比例)'])
df_filter_side['得分合计'] = df_filter_side['高点左侧得分'] + df_filter_side[
'高点右侧得分'] + df_filter_side['今日左侧得分']
order = [
'股票代码', '得分合计', '高点左侧得分', '高点右侧得分', '今日左侧得分', '收盘价低于(高低点黄金比例)'
]
df_filter_side = df_filter_side[order]
df_filter_side = df_filter_side.sort_values('得分合计', ascending=False)
df_filter_side.reset_index(drop=True, inplace=True)
wx.info("[Filter_Curve Side] Completed Filter the Left & Right Side ")
return df_filter_side
def rt_cmp_big_baseline(self,
date_str='',
begin_time_stamp=0,
rt=None,
big_bl_df=None):
wx = lg.get_handle()
rt_dict_df = rt.rt_dict_df
# if date_str is None or len(date_str) == 0:
# date_str = (date.today()).strftime('%Y%m%d')
if rt_dict_df is None:
wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] 实时数据字典 是空，退出")
return None
# 暂不需要 big_bl_df 大单基线数据，不需要做对比，直接体现在BI图形上
# if big_bl_df is None:
# wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] 基线数据 是空，退出")
# return None
[cu_begin_stamp, cu_begin_time_str
] = self.rt_df_find_start_stamp(rt_stamp=begin_time_stamp)
cu_end_time_str = self.t_frame_dict.get(cu_begin_time_str)[0]
cu_t_frame = cu_begin_time_str + '-' + cu_end_time_str
rt_big_deal_df = pd.DataFrame()
for id in rt_dict_df.keys():
if rt_dict_df[id] is None:
wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] {} 未产生实时交易数据，进入下一支股票".
format(id))
continue
if date_str is None or len(date_str) == 0: # 处理当天数据
date_str = (date.today()).strftime('%Y%m%d')
# else: # 处理历史数据 , 只用来做调试
# 起始时间边界对齐
# 从记录文件读取上一次查询实时交易的截止时间，本次查询的开始时间
# 获得的时间戳有两种情况 1）取整（半小时）的时间戳 2）空文件，自动处理成 09:25
# 记录文件名：日期_数据来源
# frame_begin_time_str = '09:25'
# frame_begin_stamp = int(time.mktime(time.strptime(date_str + frame_begin_time_str, '%Y%m%d%H:%M')))
# 起始时间边界对齐
if begin_time_stamp == 0:
[frame_begin_stamp,
frame_begin_time_str] = self.rt_df_find_start_stamp(
rt_stamp=rt_dict_df[id]['time_stamp'].min())
else:
[frame_begin_stamp, frame_begin_time_str
] = self.rt_df_find_start_stamp(rt_stamp=begin_time_stamp)
end_stamp = rt_dict_df[id].time_stamp.max()
# 按时间段 切片rt数据，计算每个片段的 大单数据，并与基线做比对，再导入基线数据库
while frame_begin_stamp < end_stamp:
# frame_begin_time_str = time.strftime("%H:%M", time.localtime(frame_begin_stamp))
frame_end_time_str = self.t_frame_dict.get(
frame_begin_time_str)[0]
if frame_end_time_str is None:
wx.info(
"[Rt_Ana][Rt_Cmp_Big_Baseline] {} [{}] 起始时间不属于正常范围！！！！"
.format(id, frame_begin_time_str))
break
else:
t_frame = frame_begin_time_str + "-" + frame_end_time_str
frame_end_stamp = int(
time.mktime(
time.strptime(date_str + frame_end_time_str,
'%Y%m%d%H:%M')))
# 采集的实时数据未到 该时间段 结束，仍然继续统计 该时间段内的情况，下面的判断屏蔽掉啦
# if frame_end_stamp > end_stamp:
# wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] {} {} 已超出本次获取的实时数据范围，进入下一支股票".format(id, t_frame))
# break
if frame_end_time_str == '15:00': # 15:00 收市后，有最后一笔交易记录 产生在 15:00 之后若干秒
rt_df = rt.rt_dict_df[id].loc[(
rt.rt_dict_df[id]['time_stamp'] >=
frame_begin_stamp)].copy()
else:
rt_df = rt.rt_dict_df[id].loc[
(rt.rt_dict_df[id]['time_stamp'] >= frame_begin_stamp)
& (rt.rt_dict_df[id]['time_stamp'] < frame_end_stamp
)].copy()
if rt_df is None or rt_df.empty:
wx.info(
"[Rt_Ana][Rt_Cmp_Big_Baseline] [{}] 在[{}]期间交易数据为空，开始处理下一支股票"
.format(id, t_frame))
break
# ID 的所有成交量
rt_df['amount'] = rt_df['vol'] * rt_df['price']
rt_amount = rt_df['amount'].sum()
rt_df['io_amount'] = rt_df['amount'] * rt_df['type']
# 内外盘、中性盘的数量统计、金额统计
rt_sell_qty = rt_df.loc[rt_df["type"] == -1].shape[0]
rt_buy_qty = rt_df.loc[rt_df["type"] == 1].shape[0]
rt_air_qty = rt_df.loc[rt_df["type"] == 0].shape[0]
rt_buy_amount = rt_df.loc[rt_df["type"] == 1].amount.sum()
rt_sell_amount = rt_df.loc[rt_df["type"] == -1].amount.sum()
rt_air_amount = rt_df.loc[rt_df["type"] == 0].amount.sum()
# 成交明细中的 大单列表
rt_big_df = rt_df.loc[rt_df['amount'] >= self.rt_big_amount, ]
# 大单的数量
rt_big_qty = len(rt_big_df)
# 大单买入、卖出金额合计
rt_big_amount_sum_abs = rt_big_df['amount'].sum()
# 大单买入 卖出对冲后的金额
rt_big_amount_sum_io = rt_big_df['io_amount'].sum()
# 大单金额 占 总成交量的比例
big_abs_amount_pct = rt_big_amount_sum_abs / rt_amount
# 大单净买入 占 总成交量的比例
big_io_amount_pct = rt_big_amount_sum_io / rt_amount
# 平均每分钟的 大单买入、卖出金额
# rt_ave_big_amount_per_min_abs = rt_big_amount_sum_abs/((rt_end_time-rt_begin_time)/60)
# 卖盘的 大单金额
rt_big_sell_df = rt_big_df.loc[(rt_big_df['type'] < 0), ]
rt_big_sell_amount = rt_big_sell_df['amount'].sum()
rt_big_sell_amount_pct = rt_big_sell_amount / rt_amount
# 买盘的 大单金额
rt_big_buy_df = rt_big_df.loc[(rt_big_df['type'] > 0), ]
rt_big_buy_amount = rt_big_buy_df['amount'].sum()
rt_big_buy_amount_pct = rt_big_buy_amount / rt_amount
rt_data = {
"id": id,
"date": date_str,
"t_frame": t_frame,
"big_qty": rt_big_qty,
"big_abs_pct": big_abs_amount_pct,
"big_io_pct": big_io_amount_pct,
"big_buy_pct": rt_big_buy_amount_pct,
"big_sell_pct": rt_big_sell_amount_pct,
"amount": rt_amount,
"sell_qty": rt_sell_qty,
"sell_amount": rt_sell_amount,
"buy_qty": rt_buy_qty,
"buy_amount": rt_buy_amount,
"air_qty": rt_air_qty,
"air_amount": rt_air_amount
}
if rt_big_deal_df is None or rt_big_deal_df.empty:
rt_big_deal_df = pd.DataFrame([rt_data])
else:
rt_big_deal_df = rt_big_deal_df.append(
pd.DataFrame([rt_data]))
wx.info(
"[Rt_Ana][Rt_Cmp_Big_Baseline] [{}] 时间段[{}] 数据处理完毕".format(
id, t_frame))
# 准备进入下一个循环
frame_begin_time_str = self.t_frame_dict.get(
frame_begin_time_str)[1]
if len(frame_begin_time_str) == 0:
wx.info(
"[Rt_Ana][Rt_Cmp_Big_Baseline] {} {} 已处理完毕，进入下一支股票".
format(id, t_frame))
break
frame_begin_stamp = int(
time.mktime(
time.strptime(date_str + frame_begin_time_str,
'%Y%m%d%H:%M')))
cu_big_df = self.db.get_cu_big_deal_date(date_str=date_str,
t_frame=cu_t_frame)
if cu_big_df is None:
id_arr = list(rt_dict_df.keys())
cu_big_df = pd.DataFrame({
'id': list(id_arr),
'cu_big_qty': [float(0)] * len(id_arr),
'cu_amount': [float(0)] * len(id_arr),
'cu_sell_qty': [float(0)] * len(id_arr),
'cu_sell_amount': [float(0)] * len(id_arr),
'cu_buy_qty': [float(0)] * len(id_arr),
'cu_buy_amount': [float(0)] * len(id_arr)
})
final_cu_big_deal_df = pd.DataFrame()
for each_id in rt_big_deal_df.groupby(rt_big_deal_df['id']):
df_each_id = each_id[1].sort_values(by="t_frame", ascending=True)
df_each_id['cu_big_qty'] = df_each_id['big_qty'].cumsum() + \
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_big_qty'].values[0]
df_each_id['cu_amount'] = df_each_id['amount'].cumsum()+ \
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_amount'].values[0]
df_each_id['cu_sell_qty'] = df_each_id['sell_qty'].cumsum()+\
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_sell_qty'].values[0]
df_each_id['cu_sell_amount'] = df_each_id['sell_amount'].cumsum()+ \
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_sell_amount'].values[0]
df_each_id['cu_buy_qty'] = df_each_id['buy_qty'].cumsum()+ \
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_buy_qty'].values[0]
df_each_id['cu_buy_amount'] = df_each_id['buy_amount'].cumsum()+ \
cu_big_df.loc[cu_big_df['id']==each_id[0]]['cu_buy_amount'].values[0]
df_each_id['cu_air_qty'] = df_each_id['air_qty'].cumsum()
df_each_id['cu_air_amount'] = df_each_id['air_amount'].cumsum()
if final_cu_big_deal_df is None or len(final_cu_big_deal_df) == 0:
final_cu_big_deal_df = df_each_id
else:
final_cu_big_deal_df = final_cu_big_deal_df.append(df_each_id)
cols = [
'id', 'date', 't_frame', 'big_qty', 'big_abs_pct', 'big_io_pct',
'big_buy_pct', 'big_sell_pct', 'amount', 'sell_qty', 'sell_amount',
'buy_qty', 'buy_amount', 'air_qty', 'air_amount', 'cu_big_qty',
'cu_amount', 'cu_sell_qty', 'cu_sell_amount', 'cu_buy_qty',
'cu_buy_amount', 'cu_air_qty', 'cu_air_amount'
]
final_cu_big_deal_df = final_cu_big_deal_df.loc[:, cols]
final_cu_big_deal_df.fillna(0, inplace=True)
final_cu_big_deal_df.reset_index(drop=True, inplace=True)
wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] [{}] 导入 [{}] 条 大单、内外盘记录".format(
frame_end_time_str, len(final_cu_big_deal_df)))
return final_cu_big_deal_df
"""
if rt_big_deal_df is None or rt_big_deal_df.empty:
wx.info("[Rt_Ana][Rt_Cmp_Big_Baseline] 大单数据为空，退出")
return None
else:
cols = ['id','date','t_frame','big_qty','big_abs_pct','big_io_pct','big_buy_pct','big_sell_pct',
'amount','sell_qty','sell_amount','buy_qty','buy_amount','air_qty','air_amount']
rt_big_deal_df = rt_big_deal_df.loc[:,cols]
rt_big_deal_df.fillna(0,inplace=True)
rt_big_deal_df.reset_index(drop=True, inplace=True)
return rt_big_deal_df
"""
# 导入基线数据库
# self.db.db_load_into_RT_BL_Big_Deal(df=rt_big_deal_df)
""" 不做统计对比， 直接导入 基线数据库，BI工具图形化展示
def __del__(self):
wx = lg.get_handle()
wx.info("[OBJ] Conf_handler : __del__ called")
# -*- coding: utf-8 -*-
# __author__ = "WUX"
# dev version
import new_logger as lg
lg._init_()
wx = lg.get_handle()
from stock_package import ts_data
from realtime_package import rt_163, rt_bl, rt_ana, rt_east
# import pandas as pd
# from assess_package import back_trader
from functions import *
from conf import conf_handler, xl_handler
# 从配置文件 rt_analyer.conf 读取参数，初始化
h_conf = conf_handler(conf="rt_analyer.conf")
rt_delay = int(h_conf.rd_opt('general', 'rt_delay'))
# 读取 accounts.xlsx
xl_acc = xl_handler(f_name="accounts.xlsx")
xl_acc.rd_accounts_file()
id_arr = xl_acc.get_stock_id_from_conf()
#读取 keywords.xlsx
xl_keywords = xl_handler(f_name="keywords.xlsx")
keywords_arr = xl_keywords.rd_keywords_file()
# 实时交易对象
# rt163 = rt_163(id_arr=id_arr, date_str='')
def calc(self, stock_id, fresh=True, data_src='qfq'):
wx = lg.get_handle()
if data_src == 'cq':
if re.match('002', stock_id) is not None:
t_name = self.cq_tname_002
elif re.match('00', stock_id) is not None:
t_name = self.cq_tname_00
elif re.match('30', stock_id) is not None:
t_name = self.cq_tname_30
elif re.match('60', stock_id) is not None:
t_name = self.cq_tname_60
elif re.match('68', stock_id) is not None:
t_name = self.cq_tname_68
else:
wx.info(
"[Class MA_kits: calc] failed to identify the Stock_id {}".
format(stock_id))
return None
elif data_src == 'qfq':
if re.match('002', stock_id) is not None:
t_name = self.qfq_tname_002
elif re.match('00', stock_id) is not None:
t_name = self.qfq_tname_00
elif re.match('30', stock_id) is not None:
t_name = self.qfq_tname_30
elif re.match('60', stock_id) is not None:
t_name = self.qfq_tname_60
elif re.match('68', stock_id) is not None:
t_name = self.qfq_tname_68
else:
wx.info(
"[Class MA_kits: calc] failed to identify the Stock_id {}".
format(stock_id))
return None
else:
wx.info("[Class MA_kits: calc] failed to identify the Data Src {}".
format(data_src))
return None
sql = "select id, date, close from " + t_name + " where close > 0 and id = " + stock_id + " order by date desc "
df_ma = self.db._exec_sql(sql)
df_ma.sort_values(by='date', ascending=True, inplace=True)
# MA 5 10 20 60 移动均值
for duration in self.ma_duration:
# df_ma['MA_' + duration] = pd.rolling_mean(df_ma['close'], int(duration))
df_ma['MA_' + duration] = df_ma['close'].rolling(
int(duration)).mean()
# EMA 12 26 指数移动均值
for duration in self.ema_duration:
df_ma['EMA_' +
duration] = df_ma['close'].ewm(span=int(duration)).mean()
# MACD 快线
df_ma['DIF'] = df_ma['EMA_' + self.ema_duration[0]] - df_ma[
'EMA_' + self.ema_duration[1]]
# MACD 慢线
df_ma['DEA'] = df_ma['DIF'].ewm(span=9).mean()
# Bolling 20 计算
df_ma['bolling_mid'] = df_ma['MA_' + self.bolling]
df_ma['tmp2'] = df_ma['close'].rolling(int(self.bolling)).std()
df_ma['bolling_top'] = df_ma['MA_' + self.bolling] + 2 * df_ma['tmp2']
df_ma['bolling_bottom'] = df_ma['MA_' +
self.bolling] - 2 * df_ma['tmp2']
df_ma.drop(columns=['tmp2'], inplace=True)
df_ma.drop(columns=['close'], inplace=True)
df_ma.dropna(axis=0, how="any", inplace=True)
if fresh == False:
df_ma = df_ma.iloc[-1:] # 选取DataFrame最后一行，返回的是DataFrame
return df_ma
