def share_filter(today_str=None):
"""
推送筛选列表至influxdb
"""
f = h5py.File(conf.HDF5_FILE_SCREEN, 'a')
if today_str is None:
today_str = tradetime.get_today()
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.MEASUREMENT_FILTER_SHARE)
if f[conf.SCREEN_SHARE_FILTER].get(today_str) is None:
console.write_msg(today_str + "的筛选数据不存在")
return
screen_df = tool.df_from_dataset(f[conf.SCREEN_SHARE_FILTER], today_str,
None)
screen_df[conf.HDF5_SHARE_DATE_INDEX] = bytes(today_str, encoding="utf8")
screen_df = _datetime_index(screen_df)
screen_df = screen_df.reset_index()
num = 1
for index, row in screen_df.iterrows():
screen_df.loc[index,
conf.HDF5_SHARE_DATE_INDEX] = screen_df.loc[index][
conf.HDF5_SHARE_DATE_INDEX] + datetime.timedelta(
0, num)
num += 1
screen_df = screen_df.set_index(conf.HDF5_SHARE_DATE_INDEX)
try:
influx.write_df(screen_df, conf.MEASUREMENT_FILTER_SHARE,
{"filter_date": today_str})
except Exception as er:
print(str(er))
console.write_tail()
f.close()
return
def all_classify(classify_list, init_flag=True):
"""
获取所有分类的macd与所处均线等指标(依赖分类数据聚合)
"""
f = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
# 获取classify列表
for ctype in classify_list:
for classify_name in f[ctype]:
console.write_head(conf.HDF5_OPERATE_INDEX,
conf.HDF5_RESOURCE_TUSHARE, classify_name)
for ktype in conf.HDF5_SHARE_KTYPE:
ds_name = ktype
if f[ctype][classify_name].get(ds_name) is None:
console.write_msg(classify_name + "分类聚合detail不存在")
continue
df = tool.df_from_dataset(f[ctype][classify_name], ds_name,
None)
df["close"] = df["close"].apply(lambda x: round(x, 2))
try:
index_df = one_df(df, init_flag, True)
except Exception as er:
console.write_msg("[" + classify_name + "]" + str(er))
continue
index_ds_name = conf.HDF5_INDEX_DETAIL + "_" + ktype
if init_flag is True:
tool.delete_dataset(f[ctype][classify_name], index_ds_name)
tool.merge_df_dataset(f[ctype][classify_name], index_ds_name,
index_df.reset_index())
console.write_tail()
f.close()
return
def code_classify(code_list, classify_list):
"""
按照code整理其所属的classify
"""
f = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_OTHER_CODE_CLASSIFY)
# 获取classify列表
code_classify_df = tool.init_empty_df(["date", "code", "classify"])
today_str = tradetime.get_today()
for ctype in classify_list:
for classify_name in f[ctype]:
if f[ctype][classify_name].get(conf.HDF5_CLASSIFY_DS_CODE) is None:
console.write_msg(classify_name + "的code列表不存在")
classify_df = tool.df_from_dataset(f[ctype][classify_name],
conf.HDF5_CLASSIFY_DS_CODE,
None)
for index, row in classify_df.iterrows():
code = row[0].astype(str)
if code in code_list:
code_dict = dict()
code_dict["date"] = today_str
code_dict["code"] = code
code_dict["classify"] = classify_name
code_classify_df = code_classify_df.append(
code_dict, ignore_index=True)
console.write_tail()
f.close()
f_other = h5py.File(conf.HDF5_FILE_OTHER, 'a')
tool.delete_dataset(f_other, conf.HDF5_OTHER_CODE_CLASSIFY)
tool.merge_df_dataset(f_other, conf.HDF5_OTHER_CODE_CLASSIFY,
code_classify_df)
f_other.close()
return
def code_classify(today_str=None):
"""
推送筛选出的股票相关分类
"""
f = h5py.File(conf.HDF5_FILE_OTHER, 'a')
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_OTHER_CODE_CLASSIFY)
if today_str is None:
today_str = tradetime.get_today()
if f.get(conf.HDF5_OTHER_CODE_CLASSIFY) is None:
console.write_msg("code的分类文件不存在")
return
code_classify_df = tool.df_from_dataset(f, conf.HDF5_OTHER_CODE_CLASSIFY,
None)
code_classify_df[conf.HDF5_SHARE_DATE_INDEX] = bytes(today_str,
encoding="utf8")
code_classify_df = _datetime_index(code_classify_df)
code_classify_df = code_classify_df.reset_index()
num = 1
for index, row in code_classify_df.iterrows():
code_classify_df.loc[
index, conf.HDF5_SHARE_DATE_INDEX] = code_classify_df.loc[index][
conf.HDF5_SHARE_DATE_INDEX] + datetime.timedelta(0, num)
num += 1
code_classify_df = code_classify_df.set_index(conf.HDF5_SHARE_DATE_INDEX)
try:
influx.write_df(code_classify_df, conf.MEASUREMENT_CODE_CLASSIFY, None)
except Exception as er:
print(str(er))
console.write_tail()
f.close()
return
def margin(reset_flag=False):
"""
获取沪市融资融券
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_SH_MARGINS)
path = '/' + conf.HDF5_FUNDAMENTAL_SH_MARGINS
if f.get(path) is None:
f.create_group(path)
fundamental.get_sh_margins(f[path], reset_flag)
count.show_result()
console.write_tail()
# 获取深市融资融券
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_SZ_MARGINS)
path = '/' + conf.HDF5_FUNDAMENTAL_SZ_MARGINS
if f.get(path) is None:
f.create_group(path)
fundamental.get_sz_margins(f[path], reset_flag)
count.show_result()
console.write_tail()
f.close()
return
def all_classify(classify_list, init_flag=True):
"""
整理分类的缠论k线
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
f_classify = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
# 获取classify列表
for ctype in classify_list:
for classify_name in f_classify[ctype]:
console.write_head(conf.HDF5_OPERATE_WRAP,
conf.HDF5_RESOURCE_TUSHARE, classify_name)
for ktype in conf.HDF5_SHARE_WRAP_KTYPE:
ds_name = conf.HDF5_CLASSIFY_DS_DETAIL + "_" + ktype
if f_classify[ctype][classify_name].get(ds_name) is None:
continue
share_df = tool.df_from_dataset(
f_classify[ctype][classify_name], ds_name, None)
wrap_df = one_df(share_df)
wrap_ds_name = conf.HDF5_INDEX_WRAP + "_" + ktype
if init_flag is True:
tool.delete_dataset(f_classify[ctype][classify_name],
wrap_ds_name)
if wrap_df is not None:
tool.merge_df_dataset(f_classify[ctype][classify_name],
wrap_ds_name, wrap_df)
console.write_tail()
f_classify.close()
f.close()
return
def classify_detail(classify_list):
"""
获取类别数据
"""
f = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
for ctype in classify_list:
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
ctype)
cpath = '/' + ctype
if f.get(cpath) is None:
f.create_group(cpath)
if ctype == conf.HDF5_CLASSIFY_INDUSTRY:
# 获取工业分类
classify.get_industry_classified(f[cpath])
elif ctype == conf.HDF5_CLASSIFY_CONCEPT:
# 获取概念分类
classify.get_concept_classified(f[cpath])
elif ctype == conf.HDF5_CLASSIFY_HOT:
# 获取热门分类
classify.get_hot_classified(f[cpath])
count.show_result()
console.write_tail()
f.close()
return
def bitmex(symbol, bin_size, count):
"""
bitmex期货数据
"""
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_BITMEX,
symbol + '-' + bin_size)
f = h5py.File(conf.HDF5_FILE_BITMEX, 'a')
if f.get(symbol) is None:
f.create_group(symbol)
# 处理D,5,1数据
if bin_size in [
conf.BINSIZE_ONE_DAY,
conf.BINSIZE_ONE_HOUR,
conf.BINSIZE_FIVE_MINUTE,
conf.BINSIZE_ONE_MINUTE,
]:
df = future.history(symbol, bin_size, count)
# 处理30m数据
elif bin_size in [conf.BINSIZE_THIRTY_MINUTE, conf.BINSIZE_FOUR_HOUR]:
df = future.history_merge(symbol, bin_size, count)
if df is not None and df.empty is not True:
# 存储数据暂时不包括macd的内容
# df = df.set_index(conf.HDF5_SHARE_DATE_INDEX)
# index_df = macd.value(df)
# df = df.merge(index_df, left_index=True, right_index=True, how='left')
# df = df.dropna().reset_index()
tool.merge_df_dataset(f[symbol], bin_size, df)
f.close()
console.write_tail()
return
def all_macd_trend(code_list, start_date):
"""
整理所有股票的macd趋势数据
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_INDEX_MACD_TREND)
for code in code_list:
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f.get(code_group_path) is None:
continue
# 忽略停牌、退市、无法获取的情况
if f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_QUIT
) is not None or f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_ST) is not None:
continue
for ktype in conf.HDF5_SHARE_KTYPE:
trend_df = code_macd_trend(f[code_prefix][code], ktype)
if trend_df is not None:
ds_name = conf.HDF5_INDEX_MACD_TREND + "_" + ktype
if f[code_prefix][code].get(ds_name) is not None:
tool.delete_dataset(f[code_prefix][code], ds_name)
tool.create_df_dataset(f[code_prefix][code], ds_name, trend_df)
console.write_tail()
f.close()
return
def ipo():
"""
聚合ipo上市数据
"""
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
path = '/' + conf.HDF5_FUNDAMENTAL_IPO
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_IPO)
path = '/' + conf.HDF5_FUNDAMENTAL_IPO
ipo_sum_dict = dict()
if f.get(path) is not None:
df = tool.df_from_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO, None)
df["issue_date"] = df["issue_date"].str.decode("utf-8")
df["ipo_date"] = df["ipo_date"].str.decode("utf-8")
for index, row in df.iterrows():
trade_date = row["ipo_date"]
# 统一单位为亿元
sum_price = round(row["funds"], 2)
if trade_date in ipo_sum_dict:
ipo_sum_dict[trade_date] += sum_price
else:
ipo_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(ipo_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO_DETAIL,
sum_df)
console.write_tail()
f.close()
return
def all_classify_detail(classify_list, omit_list, start_date):
"""
遍历所有分类，聚合所有code获取分类均值
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
f_classify = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
# 获取classify列表
for ctype in classify_list:
for classify_name in f_classify[ctype]:
console.write_head(conf.HDF5_OPERATE_ARRANGE,
conf.HDF5_RESOURCE_TUSHARE, classify_name)
if f_classify[ctype][classify_name].get(
conf.HDF5_CLASSIFY_DS_CODE) is None:
console.write_msg(classify_name + "的detail文件不存在")
continue
for ktype in conf.HDF5_SHARE_KTYPE:
mean_df = one_classify_detail(
f, f_classify[ctype][classify_name].get(
conf.HDF5_CLASSIFY_DS_CODE), omit_list, ktype,
start_date)
ds_name = ktype
# 如果start_date为空，则重置该数据
if start_date is None:
tool.delete_dataset(f_classify[ctype][classify_name],
ds_name)
if mean_df is not None:
tool.merge_df_dataset(f_classify[ctype][classify_name],
ds_name, mean_df)
console.write_tail()
f_classify.close()
f.close()
return
def share_detail(code_list, reset_flag=False):
"""
将股票数据推送至influxdb
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
for code in code_list:
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f.get(code_group_path) is None:
console.write_msg(code + "目录不存在")
continue
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
code)
# 推送原始kline
measurement = conf.MEASUREMENT_SHARE
_raw_kline(f[code_prefix][code], measurement, code, reset_flag)
# 推送缠论kline
measurement = conf.MEASUREMENT_SHARE_WRAP
_wrap_kline(f[code_prefix][code], measurement, code, reset_flag)
# 推送基本面数据
measurement = conf.MEASUREMENT_SHARE_BASIC
_basic_info(f[code_prefix][code], measurement, code, reset_flag)
console.write_blank()
console.write_tail()
f.close()
return
def operate_quit(action_type):
"""
将退市quit内容，转换成标签添加在对应code下
"""
f = h5py.File(conf.HDF5_FILE_BASIC, 'a')
console.write_head(action_type, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_BASIC_QUIT)
path = '/' + conf.HDF5_BASIC_QUIT
if f.get(path) is None:
console.write_msg("quit的detail不存在")
return
quit_list = [
conf.HDF5_BASIC_QUIT_TERMINATE,
conf.HDF5_BASIC_QUIT_SUSPEND,
]
for qtype in quit_list:
# 如果文件不存在，则退出
quit_df = tool.df_from_dataset(f[path], qtype, None)
if quit_df is not None and quit_df.empty is not True:
quit_df["code"] = quit_df["code"].str.decode("utf-8")
# 将退市内容，转换成标签添加在对应code下
tool.op_attr_by_codelist(action_type, quit_df["code"].values,
conf.HDF5_BASIC_QUIT, True)
else:
console.write_msg("quit的detail数据获取失败")
console.write_tail()
f.close()
return
def xsg():
"""
聚合xsg数据
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
f_share = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_XSG)
path = '/' + conf.HDF5_FUNDAMENTAL_XSG
xsg_sum_dict = dict()
if f.get(path) is not None:
for month in f[path]:
df = tool.df_from_dataset(f[path], month, None)
df["code"] = df["code"].str.decode("utf-8")
df["count"] = df["count"].str.decode("utf-8")
df[conf.HDF5_SHARE_DATE_INDEX] = df[
conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8")
for index, row in df.iterrows():
code = row["code"]
xsg_date_str = row[conf.HDF5_SHARE_DATE_INDEX]
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f_share.get(code_group_path) is None:
continue
# 获取限售股解禁前一天的价格
share_df = tool.df_from_dataset(f_share[code_group_path], "D",
None)
share_df[conf.HDF5_SHARE_DATE_INDEX] = share_df[
conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8")
share_df = share_df.set_index(conf.HDF5_SHARE_DATE_INDEX)
share_df = share_df[:xsg_date_str]
if len(share_df) == 0:
continue
close = share_df.tail(1)["close"]
# 万股*元，统一单位为亿
code_sum = close.values * float(row["count"]) * 10000
sum_price = round(code_sum[0] / 10000 / 10000, 2)
# trade_date = tradetime.get_week_of_date(xsg_date_str, "D")
trade_date = xsg_date_str
if trade_date in xsg_sum_dict:
xsg_sum_dict[trade_date] += sum_price
else:
xsg_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(xsg_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_XSG_DETAIL, sum_df)
console.write_tail()
f_share.close()
f.close()
return
def code_detail(code_list, start_date):
"""
将code的basic内容，整理至share文件下
"""
# 获取basic所有日期的detail，并遍历读取详细信息
f = h5py.File(conf.HDF5_FILE_BASIC, 'a')
f_share = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_BASIC_DETAIL)
path = '/' + conf.HDF5_BASIC_DETAIL
if f.get(path) is None:
return
code_basic_dict = dict()
for date in f[path]:
if start_date is not None and date < start_date:
console.write_msg(start_date + "起始日期大于基本数据的最大日期")
continue
df = tool.df_from_dataset(f[path], date, None)
df["code"] = df["code"].str.decode("utf-8")
df = df.set_index("code")
for code in df.index:
if code not in code_list:
continue
if code not in code_basic_dict:
code_basic_dict[code] = tool.init_empty_df(df.columns)
code_basic_dict[code].loc[date] = df.loc[code, :]
for code, code_df in code_basic_dict.items():
code_df.index.name = conf.HDF5_SHARE_DATE_INDEX
code_df = code_df.reset_index().sort_values(
by=[conf.HDF5_SHARE_DATE_INDEX])
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f_share.get(code_group_path) is None:
console.write_msg(code + "的detail文件不存在")
continue
if start_date is None:
tool.delete_dataset(f_share[code_group_path],
conf.HDF5_BASIC_DETAIL)
tool.merge_df_dataset(f_share[code_group_path], conf.HDF5_BASIC_DETAIL,
code_df)
console.write_exec()
console.write_blank()
console.write_tail()
f_share.close()
f.close()
return
def all_share(omit_list):
"""
根据类别获取所有share数据
"""
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_RESOURCE_TUSHARE)
# 初始化相关文件
f_classify = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
code_list = []
classify_list = [
conf.HDF5_CLASSIFY_CONCEPT,
conf.HDF5_CLASSIFY_INDUSTRY,
conf.HDF5_CLASSIFY_HOT,
]
for ctype in classify_list:
# 获取概念下具体类别名称
for classify_name in f_classify[ctype]:
# 如果不存在code list，则跳过
if f_classify[ctype][classify_name].get(
conf.HDF5_CLASSIFY_DS_CODE) is None:
continue
# 获取单个分类(code的list)下的share数据
for row in f_classify[ctype][classify_name][
conf.HDF5_CLASSIFY_DS_CODE]:
code = row[0].astype(str)
if code not in code_list:
code_list.append(code)
f_classify.close()
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
# 初始化error记录
error.init_batch(conf.HDF5_ERROR_SHARE_GET)
for code in code_list:
code_prefix = code[0:3]
if code_prefix in omit_list:
continue
# TODO, 筛选错误数据
# history = error.get_file()
# error_history = list()
# if history is not None:
# history["ktype"] = history["ktype"].str.decode("utf-8")
# history["code"] = history["code"].str.decode("utf-8")
# error_history = history.values
code_share(f, code)
# 记录错误内容
error.write_batch()
# 输出获取情况
count.show_result()
f.close()
console.write_tail()
return
def xsg():
"""
获取限售股解禁数据
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_XSG)
path = '/' + conf.HDF5_FUNDAMENTAL_XSG
if f.get(path) is None:
f.create_group(path)
fundamental.get_xsg(f[path])
count.show_result()
console.write_tail()
f.close()
return
def ipo(reset_flag=False):
"""
获取ipo数据
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_IPO)
path = '/' + conf.HDF5_FUNDAMENTAL_IPO
if f.get(path) is None:
f.create_group(path)
fundamental.get_ipo(f[path], reset_flag)
count.show_result()
console.write_tail()
f.close()
return
def basic_detail():
"""
聚合xsg、ipo、shm、szm等数据，推送至influxdb
"""
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
# 获取xsg
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_XSG)
if f.get(conf.HDF5_FUNDAMENTAL_XSG_DETAIL) is not None:
xsg_df = tool.df_from_dataset(f, conf.HDF5_FUNDAMENTAL_XSG_DETAIL,
None)
xsg_df = _datetime_index(xsg_df)
influx.reset_df(xsg_df, conf.MEASUREMENT_BASIC, {"btype": "xsg"})
console.write_tail()
# 获取ipo
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_IPO)
if f.get(conf.HDF5_FUNDAMENTAL_IPO) and f[conf.HDF5_FUNDAMENTAL_IPO].get(
conf.HDF5_FUNDAMENTAL_IPO_DETAIL) is not None:
ipo_df = tool.df_from_dataset(f[conf.HDF5_FUNDAMENTAL_IPO],
conf.HDF5_FUNDAMENTAL_IPO_DETAIL, None)
ipo_df = _datetime_index(ipo_df)
influx.reset_df(ipo_df, conf.MEASUREMENT_BASIC, {"btype": "ipo"})
console.write_tail()
# 获取sh融资融券
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_SH_MARGINS)
console.write_tail()
if f.get(conf.HDF5_FUNDAMENTAL_SH_MARGINS) and f[
conf.HDF5_FUNDAMENTAL_SH_MARGINS].get(
conf.HDF5_FUNDAMENTAL_SH_MARGINS_DETAIL) is not None:
shm_df = tool.df_from_dataset(f[conf.HDF5_FUNDAMENTAL_SH_MARGINS],
conf.HDF5_FUNDAMENTAL_SH_MARGINS_DETAIL,
None)
shm_df = _datetime_index(shm_df)
influx.reset_df(shm_df, conf.MEASUREMENT_BASIC, {"btype": "shm"})
# 获取sz融资融券
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_SZ_MARGINS)
if f.get(conf.HDF5_FUNDAMENTAL_SZ_MARGINS) and f[
conf.HDF5_FUNDAMENTAL_SZ_MARGINS].get(
conf.HDF5_FUNDAMENTAL_SZ_MARGINS_DETAIL) is not None:
shz_df = tool.df_from_dataset(f[conf.HDF5_FUNDAMENTAL_SZ_MARGINS],
conf.HDF5_FUNDAMENTAL_SZ_MARGINS_DETAIL,
None)
shz_df = _datetime_index(shz_df)
influx.reset_df(shz_df, conf.MEASUREMENT_BASIC, {"btype": "szm"})
console.write_tail()
f.close()
return
def index_detail(reset_flag=False):
"""
将指数数据推送至influxdb
"""
f = h5py.File(conf.HDF5_FILE_INDEX, 'a')
for code in f:
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
code)
# 推送原始kline
measurement = conf.MEASUREMENT_INDEX
_raw_kline(f[code], measurement, code, reset_flag)
# 推送缠论kline
measurement = conf.MEASUREMENT_INDEX_WRAP
_wrap_kline(f[code], measurement, code, reset_flag)
console.write_blank()
console.write_tail()
f.close()
return
def index_share():
"""
获取对应指数数据
"""
index_list = ["sh", "sz", "hs300", "sz50", "zxb", "cyb"]
# 初始化相关文件
f = h5py.File(conf.HDF5_FILE_INDEX, 'a')
for code in index_list:
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
code)
code_group_path = '/' + code
if f.get(code_group_path) is None:
f.create_group(code_group_path)
# 获取不同周期的数据
for ktype in conf.HDF5_SHARE_KTYPE:
share.get_share_data(code, f[code_group_path], ktype,
share.INDEX_TYPE)
console.write_tail()
f.close()
return
def st():
"""
获取风险警示板列表
"""
# 删除以往添加的标签
arrange.operate_st(conf.HDF5_OPERATE_DEL)
# 获取最新的数据
f = h5py.File(conf.HDF5_FILE_BASIC, 'a')
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_BASIC_ST)
path = '/' + conf.HDF5_BASIC_ST
if f.get(path) is None:
f.create_group(path)
basic.get_st(f[path])
console.write_tail()
f.close()
# 添加标签
arrange.operate_st(conf.HDF5_OPERATE_ADD)
return
def all_index(init_flag=True):
"""
整理指数的缠论k线
"""
f = h5py.File(conf.HDF5_FILE_INDEX, 'a')
for code in f:
console.write_head(conf.HDF5_OPERATE_WRAP, conf.HDF5_RESOURCE_TUSHARE,
code)
for ktype in conf.HDF5_SHARE_WRAP_KTYPE:
if f[code].get(ktype) is None:
continue
share_df = tool.df_from_dataset(f[code], ktype, None)
wrap_df = one_df(share_df)
wrap_ds_name = conf.HDF5_INDEX_WRAP + "_" + ktype
if init_flag is True:
tool.delete_dataset(f[code], wrap_ds_name)
if wrap_df is not None:
tool.merge_df_dataset(f[code], wrap_ds_name, wrap_df)
console.write_tail()
f.close()
return
def all_share(omit_list, init_flag=True):
"""
获取所有股票的macd与所处均线等指标
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
for code_prefix in f:
if code_prefix in omit_list:
continue
console.write_head(conf.HDF5_OPERATE_INDEX, conf.HDF5_RESOURCE_TUSHARE,
code_prefix)
for code in f[code_prefix]:
# 忽略停牌、退市、无法获取的情况
if f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_QUIT) is not None:
continue
if f[code_prefix][code].attrs.get(conf.HDF5_BASIC_ST) is not None:
continue
code_group_path = '/' + code_prefix + '/' + code
for ktype in conf.HDF5_SHARE_KTYPE:
try:
if f.get(code_group_path) is None or f[code_prefix][
code].get(ktype) is None:
console.write_msg(code + "-" + ktype + "的detail不存在")
continue
df = tool.df_from_dataset(f[code_prefix][code], ktype,
None)
index_df = one_df(df, init_flag)
ds_name = conf.HDF5_INDEX_DETAIL + "_" + ktype
if init_flag is True:
tool.delete_dataset(f[code_prefix][code], ds_name)
tool.merge_df_dataset(f[code_prefix][code], ds_name,
index_df.reset_index())
except Exception as er:
print(str(er))
console.write_exec()
console.write_blank()
console.write_tail()
f.close()
return
def margins(mtype):
"""
聚合融资融券数据
"""
if mtype == "sh":
mtype_index = conf.HDF5_FUNDAMENTAL_SH_MARGINS
mtype_index_detail = conf.HDF5_FUNDAMENTAL_SH_MARGINS_DETAIL
elif mtype == "sz":
mtype_index = conf.HDF5_FUNDAMENTAL_SZ_MARGINS
mtype_index_detail = conf.HDF5_FUNDAMENTAL_SZ_MARGINS_DETAIL
else:
print("mtype " + mtype + " error\r\n")
return
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
path = '/' + mtype_index
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
mtype_index)
console.write_tail()
margin_sum_dict = dict()
if f.get(path) is not None:
df = tool.df_from_dataset(f[path], mtype_index, None)
df["opDate"] = df["opDate"].str.decode("utf-8")
for index, row in df.iterrows():
# trade_date = tradetime.get_week_of_date(row["opDate"], "D")
trade_date = row["opDate"]
# 统一单位为亿元
sum_price = round((row["rzmre"] - row["rqmcl"]) / 10000 / 10000, 2)
if trade_date in margin_sum_dict:
margin_sum_dict[trade_date] += sum_price
else:
margin_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(margin_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f[mtype_index], mtype_index_detail, sum_df)
f.close()
return
def all_index(init_flag=True):
"""
处理所有指数的均线与macd
"""
f = h5py.File(conf.HDF5_FILE_INDEX, 'a')
for code in f:
console.write_head(conf.HDF5_OPERATE_INDEX, conf.HDF5_RESOURCE_TUSHARE,
code)
for ktype in conf.HDF5_SHARE_KTYPE:
if f[code].get(ktype) is None:
console.write_msg(code + "-" + ktype + "的detail不存在")
continue
df = tool.df_from_dataset(f[code], ktype, None)
index_df = one_df(df, init_flag)
index_ds_name = conf.HDF5_INDEX_DETAIL + "_" + ktype
if init_flag is True:
tool.delete_dataset(f[code], index_ds_name)
tool.merge_df_dataset(f[code], index_ds_name,
index_df.reset_index())
console.write_tail()
f.close()
return
def filter_share(code_list, start_date):
"""
整理筛选的股票缠论k线
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_WRAP, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_INDEX_WRAP)
for code in code_list:
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f.get(code_group_path) is None:
continue
# 忽略停牌、退市、无法获取的情况
if f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_QUIT
) is not None or f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_ST) is not None:
continue
for ktype in conf.HDF5_SHARE_WRAP_KTYPE:
ds_name = ktype
if f[code_prefix][code].get(ds_name) is None:
continue
share_df = tool.df_from_dataset(f[code_prefix][code], ds_name,
None)
wrap_df = one_df(share_df)
if wrap_df is not None:
ds_name = conf.HDF5_INDEX_WRAP + "_" + ktype
if f[code_prefix][code].get(ds_name) is not None:
tool.delete_dataset(f[code_prefix][code], ds_name)
tool.create_df_dataset(f[code_prefix][code], ds_name, wrap_df)
console.write_exec()
else:
console.write_pass()
console.write_blank()
console.write_tail()
f.close()
return
def basic_environment(start_date):
"""
获取基本面数据
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_BASIC, 'a')
# 初始化error记录
error.init_batch(conf.HDF5_ERROR_DETAIL_GET)
# 初始化打点记录
console.write_head(conf.HDF5_OPERATE_GET, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_BASIC_DETAIL)
path = '/' + conf.HDF5_BASIC_DETAIL
if f.get(path) is None:
f.create_group(path)
basic.get_detail(f[path], start_date)
# 记录错误内容
error.merge_batch()
# 展示打点结果
count.show_result()
console.write_tail()
f.close()
return
def classify_detail(classify_list, reset_flag=False):
"""
将分类推送至influxdb
"""
f = h5py.File(conf.HDF5_FILE_CLASSIFY, 'a')
# 获取classify列表
for ctype in classify_list:
console.write_head(conf.HDF5_OPERATE_PUSH, conf.HDF5_RESOURCE_TUSHARE,
ctype)
for classify_name in f[ctype]:
# 推送原始kline
measurement = conf.MEASUREMENT_CLASSIFY + "_" + ctype
_raw_kline(f[ctype][classify_name], measurement, classify_name,
reset_flag)
# 推送缠论kline
measurement = conf.MEASUREMENT_CLASSIFY_WRAP + "_" + ctype
_wrap_kline(f[ctype][classify_name], measurement, classify_name,
reset_flag)
console.write_blank()
console.write_tail()
f.close()
return
def operate_st(action_type):
"""
将st内容，转换成标签添加在对应code下
"""
f = h5py.File(conf.HDF5_FILE_BASIC, 'a')
console.write_head(action_type, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_BASIC_ST)
path = '/' + conf.HDF5_BASIC_ST
# 如果文件不存在，则退出
if f.get(path) is None:
console.write_msg("st的detail文件不存在")
return
st_df = tool.df_from_dataset(f[path], conf.HDF5_BASIC_ST, None)
if st_df is not None and st_df.empty is not True:
st_df["code"] = st_df["code"].str.decode("utf-8")
# 将st内容，转换成标签添加在对应code下
tool.op_attr_by_codelist(action_type, st_df["code"].values,
conf.HDF5_BASIC_ST, True)
else:
console.write_msg("st的detail数据获取失败")
console.write_tail()
f.close()
return
