def get_xsg(f): """ 获取限售股解禁 """ for year in range(2010, datetime.today().year + 1): for month in range(1, 13): if month in range(1, 10): dset_name = str(year) + "0" + str(month) else: dset_name = str(year) + str(month) if f.get(dset_name) is not None: count.inc_by_index(conf.HDF5_COUNT_PASS) continue try: df = ts.xsg_data(year=year, month=month, pause=conf.REQUEST_BLANK) df = df.drop("name", axis=1) df = df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX]) tool.create_df_dataset(f, dset_name, df) console.write_exec() count.inc_by_index(conf.HDF5_COUNT_GET) except Exception as er: print(str(er)) time.sleep(conf.REQUEST_BLANK) return
def all_macd_trend(code_list, start_date): """ 整理所有股票的macd趋势数据 """ f = h5py.File(conf.HDF5_FILE_SHARE, 'a') console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE, conf.HDF5_INDEX_MACD_TREND) for code in code_list: code_prefix = code[0:3] code_group_path = '/' + code_prefix + '/' + code if f.get(code_group_path) is None: continue # 忽略停牌、退市、无法获取的情况 if f[code_prefix][code].attrs.get( conf.HDF5_BASIC_QUIT ) is not None or f[code_prefix][code].attrs.get( conf.HDF5_BASIC_ST) is not None: continue for ktype in conf.HDF5_SHARE_KTYPE: trend_df = code_macd_trend(f[code_prefix][code], ktype) if trend_df is not None: ds_name = conf.HDF5_INDEX_MACD_TREND + "_" + ktype if f[code_prefix][code].get(ds_name) is not None: tool.delete_dataset(f[code_prefix][code], ds_name) tool.create_df_dataset(f[code_prefix][code], ds_name, trend_df) console.write_tail() f.close() return
def ipo(): """ 聚合ipo上市数据 """ f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a') path = '/' + conf.HDF5_FUNDAMENTAL_IPO console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE, conf.HDF5_FUNDAMENTAL_IPO) path = '/' + conf.HDF5_FUNDAMENTAL_IPO ipo_sum_dict = dict() if f.get(path) is not None: df = tool.df_from_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO, None) df["issue_date"] = df["issue_date"].str.decode("utf-8") df["ipo_date"] = df["ipo_date"].str.decode("utf-8") for index, row in df.iterrows(): trade_date = row["ipo_date"] # 统一单位为亿元 sum_price = round(row["funds"], 2) if trade_date in ipo_sum_dict: ipo_sum_dict[trade_date] += sum_price else: ipo_sum_dict[trade_date] = sum_price sum_df = tool.init_df(list(ipo_sum_dict.items()), [conf.HDF5_SHARE_DATE_INDEX, "sum"]) if len(sum_df) > 0: sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX]) tool.create_df_dataset(f[path], conf.HDF5_FUNDAMENTAL_IPO_DETAIL, sum_df) console.write_tail() f.close() return
def get_st(f): # 获取风险警示板的股票列表 if f.get(conf.HDF5_BASIC_ST) is not None: del f[conf.HDF5_BASIC_ST] df = ts.get_st_classified() df = df.drop("name", axis=1) tool.create_df_dataset(f, conf.HDF5_BASIC_ST, df) return
def merge_batch(): global index if index in batch_dict and batch_dict[index].empty is not True: f = h5py.File(conf.HDF5_FILE_ERROR) if f.get(index) is None: tool.create_df_dataset(f, index, batch_dict[index]) else: tool.merge_df_dataset(f, index, batch_dict[index]) batch_dict[index].drop(batch_dict[index].index, inplace=True) return
def get_sz_margins(f, reset_flag=False): """ 获取深市的融资融券 """ df = ts.sz_margins(pause=conf.REQUEST_BLANK) df = df.sort_values(by=["opDate"]) if reset_flag is False: tool.merge_df_dataset(f, conf.HDF5_FUNDAMENTAL_SZ_MARGINS, df) else: tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_SZ_MARGINS, df) return
def xsg(): """ 聚合xsg数据 """ # 初始化文件 f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a') f_share = h5py.File(conf.HDF5_FILE_SHARE, 'a') console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE, conf.HDF5_FUNDAMENTAL_XSG) path = '/' + conf.HDF5_FUNDAMENTAL_XSG xsg_sum_dict = dict() if f.get(path) is not None: for month in f[path]: df = tool.df_from_dataset(f[path], month, None) df["code"] = df["code"].str.decode("utf-8") df["count"] = df["count"].str.decode("utf-8") df[conf.HDF5_SHARE_DATE_INDEX] = df[ conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8") for index, row in df.iterrows(): code = row["code"] xsg_date_str = row[conf.HDF5_SHARE_DATE_INDEX] code_prefix = code[0:3] code_group_path = '/' + code_prefix + '/' + code if f_share.get(code_group_path) is None: continue # 获取限售股解禁前一天的价格 share_df = tool.df_from_dataset(f_share[code_group_path], "D", None) share_df[conf.HDF5_SHARE_DATE_INDEX] = share_df[ conf.HDF5_SHARE_DATE_INDEX].str.decode("utf-8") share_df = share_df.set_index(conf.HDF5_SHARE_DATE_INDEX) share_df = share_df[:xsg_date_str] if len(share_df) == 0: continue close = share_df.tail(1)["close"] # 万股*元,统一单位为亿 code_sum = close.values * float(row["count"]) * 10000 sum_price = round(code_sum[0] / 10000 / 10000, 2) # trade_date = tradetime.get_week_of_date(xsg_date_str, "D") trade_date = xsg_date_str if trade_date in xsg_sum_dict: xsg_sum_dict[trade_date] += sum_price else: xsg_sum_dict[trade_date] = sum_price sum_df = tool.init_df(list(xsg_sum_dict.items()), [conf.HDF5_SHARE_DATE_INDEX, "sum"]) if len(sum_df) > 0: sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX]) tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_XSG_DETAIL, sum_df) console.write_tail() f_share.close() f.close() return
def get_ipo(f, reset_flag=False): """ 获取ipo数据 """ df = ts.new_stocks(pause=conf.REQUEST_BLANK) df = df.drop("name", axis=1) df = df.sort_values(by=["ipo_date"]) if reset_flag is False: tool.merge_df_dataset(f, conf.HDF5_FUNDAMENTAL_IPO, df) else: tool.create_df_dataset(f, conf.HDF5_FUNDAMENTAL_IPO, df) return
def get_quit(f): # 获取终止上市的股票列表 if f.get(conf.HDF5_BASIC_QUIT_TERMINATE) is not None: del f[conf.HDF5_BASIC_QUIT_TERMINATE] df = ts.get_terminated() df = df.drop("name", axis=1) tool.create_df_dataset(f, conf.HDF5_BASIC_QUIT_TERMINATE, df) # 获取暂定上市的股票列表 if f.get(conf.HDF5_BASIC_QUIT_SUSPEND) is not None: del f[conf.HDF5_BASIC_QUIT_SUSPEND] df = ts.get_suspended() df = df.drop("name", axis=1) tool.create_df_dataset(f, conf.HDF5_BASIC_QUIT_SUSPEND, df) return
def _add_data(code, ktype, f, end_date, stype): df = ts.get_hist_data(code, ktype=ktype, pause=conf.REQUEST_BLANK, end=end_date) time.sleep(conf.REQUEST_BLANK) if df is not None and df.empty is not True: if stype == SHARE_TYPE: df = df[SHARE_COLS] elif stype == INDEX_TYPE: df = df[INDEX_COLS] df = df.reset_index().sort_values(by=[conf.HDF5_SHARE_DATE_INDEX]) tool.create_df_dataset(f, ktype, df) console.write_exec() count.inc_by_index(ktype) else: error.add_row([ktype, code]) count.inc_by_index("empty") return
def margins(mtype): """ 聚合融资融券数据 """ if mtype == "sh": mtype_index = conf.HDF5_FUNDAMENTAL_SH_MARGINS mtype_index_detail = conf.HDF5_FUNDAMENTAL_SH_MARGINS_DETAIL elif mtype == "sz": mtype_index = conf.HDF5_FUNDAMENTAL_SZ_MARGINS mtype_index_detail = conf.HDF5_FUNDAMENTAL_SZ_MARGINS_DETAIL else: print("mtype " + mtype + " error\r\n") return f = h5py.File(conf.HDF5_FILE_FUNDAMENTAL, 'a') path = '/' + mtype_index console.write_head(conf.HDF5_OPERATE_ARRANGE, conf.HDF5_RESOURCE_TUSHARE, mtype_index) console.write_tail() margin_sum_dict = dict() if f.get(path) is not None: df = tool.df_from_dataset(f[path], mtype_index, None) df["opDate"] = df["opDate"].str.decode("utf-8") for index, row in df.iterrows(): # trade_date = tradetime.get_week_of_date(row["opDate"], "D") trade_date = row["opDate"] # 统一单位为亿元 sum_price = round((row["rzmre"] - row["rqmcl"]) / 10000 / 10000, 2) if trade_date in margin_sum_dict: margin_sum_dict[trade_date] += sum_price else: margin_sum_dict[trade_date] = sum_price sum_df = tool.init_df(list(margin_sum_dict.items()), [conf.HDF5_SHARE_DATE_INDEX, "sum"]) if len(sum_df) > 0: sum_df = sum_df.sort_values(by=[conf.HDF5_SHARE_DATE_INDEX]) tool.create_df_dataset(f[mtype_index], mtype_index_detail, sum_df) f.close() return
def filter_share(code_list, start_date): """ 整理筛选的股票缠论k线 """ f = h5py.File(conf.HDF5_FILE_SHARE, 'a') console.write_head(conf.HDF5_OPERATE_WRAP, conf.HDF5_RESOURCE_TUSHARE, conf.HDF5_INDEX_WRAP) for code in code_list: code_prefix = code[0:3] code_group_path = '/' + code_prefix + '/' + code if f.get(code_group_path) is None: continue # 忽略停牌、退市、无法获取的情况 if f[code_prefix][code].attrs.get( conf.HDF5_BASIC_QUIT ) is not None or f[code_prefix][code].attrs.get( conf.HDF5_BASIC_ST) is not None: continue for ktype in conf.HDF5_SHARE_WRAP_KTYPE: ds_name = ktype if f[code_prefix][code].get(ds_name) is None: continue share_df = tool.df_from_dataset(f[code_prefix][code], ds_name, None) wrap_df = one_df(share_df) if wrap_df is not None: ds_name = conf.HDF5_INDEX_WRAP + "_" + ktype if f[code_prefix][code].get(ds_name) is not None: tool.delete_dataset(f[code_prefix][code], ds_name) tool.create_df_dataset(f[code_prefix][code], ds_name, wrap_df) console.write_exec() else: console.write_pass() console.write_blank() console.write_tail() f.close() return
def get_detail(f, start_date): # 按日间隔获取 if start_date is None: start_date = datetime.strptime("2016-08-09", "%Y-%m-%d") # 获取历史错误数据 history = error.get_file() close_history = list() if history is not None: history["type"] = history["type"].str.decode("utf-8") history["date"] = history["date"].str.decode("utf-8") close_history = history[history["type"] == "close"]["date"].values while start_date <= datetime.now(): try: start_date_str = datetime.strftime(start_date, "%Y-%m-%d") # 如果是周六日,已获取,或者闭盘的日子则跳过 if start_date.weekday() < 5 and start_date_str not in close_history and f.get(start_date_str) is None: df = ts.get_stock_basics(start_date_str) time.sleep(conf.REQUEST_BLANK) if df is not None and df.empty is not True: df = df.drop("name", axis=1) df = df.drop("area", axis=1) df = df.drop("industry", axis=1) tool.create_df_dataset(f, start_date_str, df.reset_index()) count.inc_by_index(conf.HDF5_COUNT_GET) console.write_exec() else: count.inc_by_index(conf.HDF5_COUNT_PASS) except Exception as er: time.sleep(conf.REQUEST_BLANK) if str(er) != "HTTP Error 404: Not Found": error.add_row([GET_DETAIL_OTHER, start_date_str]) print(str(er)) else: error.add_row([GET_DETAIL_CLOSE, start_date_str]) start_date = start_date + timedelta(days=1) return