def get_xsg(f):
"""
获取限售股解禁
"""
for year in range(2010, datetime.today().year + 1):
for month in range(1, 13):
if month in range(1, 10):
dset_name = str(year) + "0" + str(month)
else:
dset_name = str(year) + str(month)
if f.get(dset_name) is not None:
count.inc_by_index(conf.HDF5_COUNT_PASS)
continue
try:
df = ts.xsg_data(year=year,
month=month,
pause=conf.REQUEST_BLANK)
df = df.drop("name", axis=1)
df = df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f, dset_name, df)
console.write_exec()
count.inc_by_index(conf.HDF5_COUNT_GET)
except Exception as er:
print(str(er))
time.sleep(conf.REQUEST_BLANK)
return
def all_macd_trend(code_list, start_date):
"""
整理所有股票的macd趋势数据
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_INDEX_MACD_TREND)
for code in code_list:
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f.get(code_group_path) is None:
continue
# 忽略停牌、退市、无法获取的情况
if f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_QUIT
) is not None or f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_ST) is not None:
continue
for ktype in conf.HDF5_SHARE_KTYPE:
trend_df = code_macd_trend(f[code_prefix][code], ktype)
if trend_df is not None:
ds_name = conf.HDF5_INDEX_MACD_TREND + "_" + ktype
if f[code_prefix][code].get(ds_name) is not None:
tool.delete_dataset(f[code_prefix][code], ds_name)
tool.create_df_dataset(f[code_prefix][code], ds_name, trend_df)
console.write_tail()
f.close()
return
def ipo():
"""
聚合ipo上市数据
"""
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
path = '/' + conf.HDF5_FUNDAMENTAL_IPO
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_IPO)
path = '/' + conf.HDF5_FUNDAMENTAL_IPO
ipo_sum_dict = dict()
if f.get(path) is not None:
df = tool.df_from_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO, None)
df["issue_date"] = df["issue_date"].str.decode("utf-8")
df["ipo_date"] = df["ipo_date"].str.decode("utf-8")
for index, row in df.iterrows():
trade_date = row["ipo_date"]
# 统一单位为亿元
sum_price = round(row["funds"], 2)
if trade_date in ipo_sum_dict:
ipo_sum_dict[trade_date] += sum_price
else:
ipo_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(ipo_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO_DETAIL,
sum_df)
console.write_tail()
f.close()
return
def get_st(f):
# 获取风险警示板的股票列表
if f.get(conf.HDF5_BASIC_ST) is not None:
del f[conf.HDF5_BASIC_ST]
df = ts.get_st_classified()
df = df.drop("name", axis=1)
tool.create_df_dataset(f, conf.HDF5_BASIC_ST, df)
return
def merge_batch():
global index
if index in batch_dict and batch_dict[index].empty is not True:
f = h5py.File(conf.HDF5_FILE_ERROR)
if f.get(index) is None:
tool.create_df_dataset(f, index, batch_dict[index])
else:
tool.merge_df_dataset(f, index, batch_dict[index])
batch_dict[index].drop(batch_dict[index].index, inplace=True)
return
def get_sz_margins(f, reset_flag=False):
"""
获取深市的融资融券
"""
df = ts.sz_margins(pause=conf.REQUEST_BLANK)
df = df.sort_values(by=["opDate"])
if reset_flag is False:
tool.merge_df_dataset(f, conf.HDF5_FUNDAMENTAL_SZ_MARGINS, df)
else:
tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_SZ_MARGINS, df)
return
def xsg():
"""
聚合xsg数据
"""
# 初始化文件
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
f_share = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_FUNDAMENTAL_XSG)
path = '/' + conf.HDF5_FUNDAMENTAL_XSG
xsg_sum_dict = dict()
if f.get(path) is not None:
for month in f[path]:
df = tool.df_from_dataset(f[path], month, None)
df["code"] = df["code"].str.decode("utf-8")
df["count"] = df["count"].str.decode("utf-8")
df[conf.HDF5_SHARE_DATE_INDEX] = df[
conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8")
for index, row in df.iterrows():
code = row["code"]
xsg_date_str = row[conf.HDF5_SHARE_DATE_INDEX]
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f_share.get(code_group_path) is None:
continue
# 获取限售股解禁前一天的价格
share_df = tool.df_from_dataset(f_share[code_group_path], "D",
None)
share_df[conf.HDF5_SHARE_DATE_INDEX] = share_df[
conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8")
share_df = share_df.set_index(conf.HDF5_SHARE_DATE_INDEX)
share_df = share_df[:xsg_date_str]
if len(share_df) == 0:
continue
close = share_df.tail(1)["close"]
# 万股*元，统一单位为亿
code_sum = close.values * float(row["count"]) * 10000
sum_price = round(code_sum[0] / 10000 / 10000, 2)
# trade_date = tradetime.get_week_of_date(xsg_date_str, "D")
trade_date = xsg_date_str
if trade_date in xsg_sum_dict:
xsg_sum_dict[trade_date] += sum_price
else:
xsg_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(xsg_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_XSG_DETAIL, sum_df)
console.write_tail()
f_share.close()
f.close()
return
def get_ipo(f, reset_flag=False):
"""
获取ipo数据
"""
df = ts.new_stocks(pause=conf.REQUEST_BLANK)
df = df.drop("name", axis=1)
df = df.sort_values(by=["ipo_date"])
if reset_flag is False:
tool.merge_df_dataset(f, conf.HDF5_FUNDAMENTAL_IPO, df)
else:
tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_IPO, df)
return
def get_quit(f):
# 获取终止上市的股票列表
if f.get(conf.HDF5_BASIC_QUIT_TERMINATE) is not None:
del f[conf.HDF5_BASIC_QUIT_TERMINATE]
df = ts.get_terminated()
df = df.drop("name", axis=1)
tool.create_df_dataset(f, conf.HDF5_BASIC_QUIT_TERMINATE, df)
# 获取暂定上市的股票列表
if f.get(conf.HDF5_BASIC_QUIT_SUSPEND) is not None:
del f[conf.HDF5_BASIC_QUIT_SUSPEND]
df = ts.get_suspended()
df = df.drop("name", axis=1)
tool.create_df_dataset(f, conf.HDF5_BASIC_QUIT_SUSPEND, df)
return
def _add_data(code, ktype, f, end_date, stype):
df = ts.get_hist_data(code, ktype=ktype, pause=conf.REQUEST_BLANK, end=end_date)
time.sleep(conf.REQUEST_BLANK)
if df is not None and df.empty is not True:
if stype == SHARE_TYPE:
df = df[SHARE_COLS]
elif stype == INDEX_TYPE:
df = df[INDEX_COLS]
df = df.reset_index().sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f, ktype, df)
console.write_exec()
count.inc_by_index(ktype)
else:
error.add_row([ktype, code])
count.inc_by_index("empty")
return
def margins(mtype):
"""
聚合融资融券数据
"""
if mtype == "sh":
mtype_index = conf.HDF5_FUNDAMENTAL_SH_MARGINS
mtype_index_detail = conf.HDF5_FUNDAMENTAL_SH_MARGINS_DETAIL
elif mtype == "sz":
mtype_index = conf.HDF5_FUNDAMENTAL_SZ_MARGINS
mtype_index_detail = conf.HDF5_FUNDAMENTAL_SZ_MARGINS_DETAIL
else:
print("mtype " + mtype + " error\r\n")
return
f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a')
path = '/' + mtype_index
console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE,
mtype_index)
console.write_tail()
margin_sum_dict = dict()
if f.get(path) is not None:
df = tool.df_from_dataset(f[path], mtype_index, None)
df["opDate"] = df["opDate"].str.decode("utf-8")
for index, row in df.iterrows():
# trade_date = tradetime.get_week_of_date(row["opDate"], "D")
trade_date = row["opDate"]
# 统一单位为亿元
sum_price = round((row["rzmre"] - row["rqmcl"]) / 10000 / 10000, 2)
if trade_date in margin_sum_dict:
margin_sum_dict[trade_date] += sum_price
else:
margin_sum_dict[trade_date] = sum_price
sum_df = tool.init_df(list(margin_sum_dict.items()),
[conf.HDF5_SHARE_DATE_INDEX, "sum"])
if len(sum_df) > 0:
sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX])
tool.create_df_dataset(f[mtype_index], mtype_index_detail, sum_df)
f.close()
return
def filter_share(code_list, start_date):
"""
整理筛选的股票缠论k线
"""
f = h5py.File(conf.HDF5_FILE_SHARE, 'a')
console.write_head(conf.HDF5_OPERATE_WRAP, conf.HDF5_RESOURCE_TUSHARE,
conf.HDF5_INDEX_WRAP)
for code in code_list:
code_prefix = code[0:3]
code_group_path = '/' + code_prefix + '/' + code
if f.get(code_group_path) is None:
continue
# 忽略停牌、退市、无法获取的情况
if f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_QUIT
) is not None or f[code_prefix][code].attrs.get(
conf.HDF5_BASIC_ST) is not None:
continue
for ktype in conf.HDF5_SHARE_WRAP_KTYPE:
ds_name = ktype
if f[code_prefix][code].get(ds_name) is None:
continue
share_df = tool.df_from_dataset(f[code_prefix][code], ds_name,
None)
wrap_df = one_df(share_df)
if wrap_df is not None:
ds_name = conf.HDF5_INDEX_WRAP + "_" + ktype
if f[code_prefix][code].get(ds_name) is not None:
tool.delete_dataset(f[code_prefix][code], ds_name)
tool.create_df_dataset(f[code_prefix][code], ds_name, wrap_df)
console.write_exec()
else:
console.write_pass()
console.write_blank()
console.write_tail()
f.close()
return
def get_detail(f, start_date):
# 按日间隔获取
if start_date is None:
start_date = datetime.strptime("2016-08-09", "%Y-%m-%d")
# 获取历史错误数据
history = error.get_file()
close_history = list()
if history is not None:
history["type"] = history["type"].str.decode("utf-8")
history["date"] = history["date"].str.decode("utf-8")
close_history = history[history["type"] == "close"]["date"].values
while start_date <= datetime.now():
try:
start_date_str = datetime.strftime(start_date, "%Y-%m-%d")
# 如果是周六日，已获取，或者闭盘的日子则跳过
if start_date.weekday() < 5 and start_date_str not in close_history and f.get(start_date_str) is None:
df = ts.get_stock_basics(start_date_str)
time.sleep(conf.REQUEST_BLANK)
if df is not None and df.empty is not True:
df = df.drop("name", axis=1)
df = df.drop("area", axis=1)
df = df.drop("industry", axis=1)
tool.create_df_dataset(f, start_date_str, df.reset_index())
count.inc_by_index(conf.HDF5_COUNT_GET)
console.write_exec()
else:
count.inc_by_index(conf.HDF5_COUNT_PASS)
except Exception as er:
time.sleep(conf.REQUEST_BLANK)
if str(er) != "HTTP Error 404: Not Found":
error.add_row([GET_DETAIL_OTHER, start_date_str])
print(str(er))
else:
error.add_row([GET_DETAIL_CLOSE, start_date_str])
start_date = start_date + timedelta(days=1)
return