def get_stock_return(self, begin_date, end_date, freq='month'):
if freq == 'week':
stock_return = DataAPI.MktEquwGet(
beginDate=begin_date,
endDate=end_date,
field=['secID', 'endDate', 'chgPct'])
stock_return = stock_return.pivot(index='endDate',
columns='secID',
values='chgPct')
stock_return.index = stock_return.index.str.replace('-', '')
elif freq == 'month':
stock_return = DataAPI.MktEqumGet(
beginDate=begin_date,
endDate=end_date,
field=['secID', 'endDate', 'chgPct'])
stock_return = stock_return.pivot(index='endDate',
columns='secID',
values='chgPct')
stock_return.index = stock_return.index.str.replace('-', '')
elif freq == 'day':
stock_return = DataAPI.MktEqudGet(
beginDate=begin_date,
endDate=end_date,
field=['secID', 'tradeDate', 'chgPct'])
stock_return = stock_return.pivot(index='tradeDate',
columns='secID',
values='chgPct')
stock_return.index = stock_return.index.str.replace('-', '')
else:
raise InputParameterError('不支持的频率格式')
return stock_return
def get_float_mkt_cap(self, trading_day_list, sec_ids):
float_mkt_cap_dict = {}
for date in trading_day_list:
df = DataAPI.MktEqudGet(secID=sec_ids,
tradeDate=date,
field=u"secID,negMarketValue",
pandas="1")
df.set_index('secID', inplace=True)
float_mkt_cap_dict[date] = df['negMarketValue']
return float_mkt_cap_dict
def get_open_price(self, trading_day_list, sec_ids):
open_price_dict = {}
for date in trading_day_list:
df = DataAPI.MktEqudGet(secID=sec_ids,
tradeDate=date,
field=u"secID,openPrice",
pandas="1")
df.set_index('secID', inplace=True)
open_price_dict[date] = df['openPrice']
return open_price_dict
def update_uqer_market(ds, **kwargs):
ref_date, _ = process_date(ds)
table = 'market'
df = api.MktEqudGet(tradeDate=ref_date)
df.rename(columns={'tradeDate': 'Date', 'ticker': 'Code'}, inplace=True)
df.Code = df.Code.astype(int)
del df['secID']
engine2.execute("delete from {0} where Date = '{1}';".format(table, ref_date))
df.to_sql(table, engine2, index=False, if_exists='append')
def get_equity_overnight_return(self, trading_day_list, sec_ids):
equity_overnight_return_dict = {}
for date in trading_day_list:
df = DataAPI.MktEqudGet(secID=sec_ids,
tradeDate=date,
field=u"secID,openPrice,preClosePrice",
pandas="1")
df['overnight_return'] = df['openPrice'] / df['preClosePrice'] - 1
df.set_index('secID', inplace=True)
equity_overnight_return_dict[date] = df['overnight_return']
return equity_overnight_return_dict
def get_equity_close_price_daily(self, universe_list, begin_date,
end_date):
df = DataAPI.MktEqudGet(secID=universe_list,
beginDate=begin_date,
endDate=end_date)
df['tradeDate'] = df['tradeDate'].apply(
lambda x: str(x).replace('-', ''))
trading_day_list = sorted(df['tradeDate'].unique())
close_price_series_dict = {}
for trading_day in trading_day_list:
close_price_series_dict[trading_day] = df[
df['tradeDate'] == trading_day].set_index(
'secID')['closePrice']
return close_price_series_dict
def get_sigle_stock_info(stock_code):
stock_ticker = stock_code
stock_infoRaw = DataAPI.MktEqudGet(secID=u"",
ticker=stock_ticker,
tradeDate=u"20200427",
beginDate=u"",
endDate=u"",
isOpen="",
field=u"",
pandas="1")
# T日交易参考价使用的是昨收
stock_info = stock_infoRaw.loc[:, [
'secID', 'ticker', 'secShortName', 'tradeDate', 'preClosePrice'
]]
single_stock_info = stock_info.loc[:, "preClosePrice"]
return single_stock_info
def get_sigle_stock_price(self, stock_code, beginDate, endDate):
'''
Args:
stock_code: str，交易代码
Returns:
float变量，一支股票当日的昨收
'''
stock_infoRaw = DataAPI.MktEqudGet(secID=u"",
ticker=stock_code,
tradeDate=u"",
beginDate=beginDate,
endDate=endDate,
isOpen="",
field=u"",
pandas="1")
# T日交易参考价使用的是昨收
return stock_infoRaw.loc[:, "preClosePrice"]
def getIC(path, today, n_days):
print('calculate IC')
filename = os.listdir(path)
filename1 = pd.Series(filename)
filename1 = filename1.apply(lambda x: x.split('.')[0])
filename1 = filename1[(filename1 > n_days) & (filename1 < today)]
trade_day = filename1.values.tolist()
col = [
'BETA', 'MOMENTUM', 'SIZE', 'EARNYILD', 'RESVOL', 'GROWTH', 'BTOP',
'LEVERAGE', 'LIQUIDTY', 'SIZENL'
]
df = pd.DataFrame([])
i = 0
for i in range(len(trade_day) - 1):
# print(i)
t = trade_day[i]
t1 = trade_day[i + 1]
ret = DataAPI.MktEqudGet(tradeDate=t1,secID=u"",ticker=u"",\
beginDate=u"",endDate=u"",isOpen="",field=['secID','chgPct'],pandas="1")
ret['secID'] = ret['secID'].apply(replaceCode)
factor = pd.read_csv(path + t + '.csv', encoding='gb18030')
factor = factor[['secID'] + col]
df_merge = pd.merge(ret, factor, on='secID')
df_merge = df_merge.set_index('secID')
df_corr = df_merge.corr('spearman')
df_corr = df_corr.iloc[0][col]
df_corr = pd.DataFrame(df_corr)
df_corr.columns = [t]
df = pd.concat([df, df_corr], axis=1)
df = df.T
df.to_csv('data/IC.csv')
print('calculate IC Done!')
def update_uqer_market(ds, **kwargs):
ref_date, this_date = process_date(ds)
flag = check_holiday(this_date)
if not flag:
return
df = api.MktEqudGet(tradeDate=ref_date)
df.rename(columns={
'tradeDate': 'trade_date',
'ticker': 'code'
},
inplace=True)
df.code = df.code.astype(int)
del df['secID']
query = delete(Market).where(Market.trade_date == this_date)
engine.execute(query)
data_info_log(df, Market)
format_data(df, format='%Y-%m-%d')
df.to_sql(Market.__table__.name, engine, index=False, if_exists='append')
def use_case_sim(security_id='002180.XSHE',
participant_ratio=0.2,
start_date='20191101',
end_date='20191231',
trade_date='2019-12-05',
passive_ratio=0.5,
mode=1):
'''
mode = 1: all execute by passive(twap/vwap); mode = 2: all execute by liquididy; mode=3: ratio of passive execute is
passive_ratio, others by liquidity
'''
prev_trade_date = '2019-12-04'
df = DataAPI.MktEqudGet(
secID=security_id,
tradeDate=u"",
beginDate=start_date,
endDate=end_date,
isOpen="",
field=["tradeDate", "chgPct", "closePrice", "turnoverVol"],
pandas="1")
df['ret_std'] = df[['chgPct']].rolling(20).apply(lambda x: x.std())
df['vol_sum'] = df[['turnoverVol']].rolling(5).apply(lambda x: x.sum())
trade_row = df[df.tradeDate == trade_date]
prev_row = df[df.tradeDate == prev_trade_date]
daily_vol = list(trade_row['ret_std'])[0]
total_vol = list(prev_row['vol_sum'])[0]
passive_ratio = 1.0 if mode == 1 else (0.0 if mode == 2 else passive_ratio)
passive_vol = total_vol * passive_ratio
liquid_vol = total_vol - passive_vol
df = DataAPI.MktBarHistDateRangeGet(securityID=security_id,
startDate=trade_date.replace('-', ''),
endDate=trade_date.replace('-', ''),
unit="1",
field=u"",
pandas="1")
# df.sort_values(by='totalVolume', ascending=True, inplace=True)
df['vwap'] = df['totalValue'] / df['totalVolume']
s = passive_vol * participant_ratio / 240
adv = total_vol
sigma = daily_vol * math.sqrt(245)
df_tick = get_features_by_date(security_id=u"002180.XSHE",
date='20191205',
min_unit="1",
tick=True)
ask_vol_sum = {}
bid_vol_sum = {}
df_tick = df_tick[df_tick.barTime <= "14:57"]
df_tick = df_tick[df_tick.barTime > '09:30']
for row in (df_tick[['barTime', 'totalAskVolume',
'totalBidVolume']].values):
_ask_tmp = ask_vol_sum.get(row[0]) or 0.0
_bid_tmp = bid_vol_sum.get(row[0]) or 0.0
ask_vol_sum.update({row[0]: _ask_tmp + row[1]})
bid_vol_sum.update({row[0]: _bid_tmp + row[2]})
q_dict = {}
for k, v in bid_vol_sum.items():
q_dict.update({k: abs(v - ask_vol_sum.get(k))})
df = df[df.barTime <= "14:57"]
df = df[df.barTime > '09:30']
mkt_vols = list(df['totalVolume'])
_tmp = [s / item for item in mkt_vols]
participant_min = dict(zip(df['barTime'], _tmp))
vt_dict = dict(zip(df['barTime'], df['totalVolume']))
total_value = 0.0
total_vol = 0
# TODO get the TWAP/VWAP simulate results (with and without market impact), check market impact results
if passive_ratio > 0:
istar, tmp_dict, perm_dict = get_market_impact(s, adv, sigma, vt_dict,
q_dict)
ret_rows = []
for bar_time, item in df[['barTime', 'vwap']].values:
total_vol += s
# _tmp_impact = tmp_dict.get(bar_time)
_tmp_impact = istar
total_value += item * s * (1 - _tmp_impact / 10000)
ret_rows.append([
bar_time, istar,
tmp_dict.get(bar_time),
perm_dict.get(bar_time)
])
mi_df = pd.DataFrame(ret_rows,
columns=['barTime', 'istar', 'tmp', 'perm'])
mi_df.to_csv('mi_report_mode_{0}_{1}.csv'.format(mode, passive_ratio))
if passive_ratio < 1.0:
print('liquid target_vol is: ', liquid_vol * participant_ratio)
liquid_trade_value, liquid_trade_vol, finish_time = get_liquid_results(
df_tick,
liquid_vol * participant_ratio,
ratio=0.13,
level=1,
price_rule=1)
print(liquid_trade_value, liquid_trade_vol,
liquid_trade_value / liquid_trade_vol, finish_time)
total_value += liquid_trade_value
total_vol += liquid_trade_vol
# liquid_value, liquid_vol, finish_time = get_liquid_results(df_tick, liquid_vol * participant_ratio, ratio=0.20,
# level=2, price_rule=1)
# print(liquid_value, liquid_value, liquid_value / liquid_vol, finish_time)
print(total_value, total_vol, total_value / total_vol)
def get_samples(mode=0, *arge, **kwargs):
''' mode 对应的测试集
0. 每种规模分别取前N，默认000300-沪深300，000001-上证综指，000905-中证500，399005-深证中小板指,399006-创业板指
1. 每个行业龙头，主要市值排行
2. 流动性看：成交量活跃度，按换手率，和成交量日间日内波动, 现为换手率，对设置的市场
3. 异动股，日间（短期）、日内
4. 风格： 动量、反转
5. 对每种指数，300，500
'''
total_num = kwargs.get('total_num') or 100
start_date = kwargs.get('start_date') or "20191201"
end_date = kwargs.get('end_date') or "20191231"
# ['000300', '000001', '000905', '399005', '399006', '399001'] #sh and sz market
# ['399005', '399006', '399001'] #sz market
tickers = kwargs.get('mkt_tickers') or ['399005', '399006', '399001']
ticker_to_exchangecd = {
'000300': 'XSHG',
'000001': 'XSHG',
'000905': 'XSHG',
'399005': 'XSHE',
'399006': 'XSHE',
'399001': 'XSHE'
}
if mode == 0:
ret = {}
_num = int(total_num / len(tickers))
for _ticker in tickers:
df = DataAPI.mIdxCloseWeightGet(secID=u"",
ticker=_ticker,
beginDate=start_date,
endDate=end_date,
field=["consID", "weight"],
pandas="1").sort_values(
by='weight', ascending=False)
# TODO choose the top weights coons, then the correlations with the becnchmark is high, change to sample num
# randomly
sec_ids = df['consID'][:_num]
ret.update({
'{0}.{1}'.format(_ticker, ticker_to_exchangecd.get(_ticker)):
sec_ids
})
elif mode == 2:
ret = {}
_num = int(total_num / len(tickers))
for _ticker in tickers:
df = DataAPI.mIdxCloseWeightGet(secID=u"",
ticker=_ticker,
beginDate=start_date,
endDate=end_date,
field=["consID", "weight"],
pandas="1").sort_values(
by='weight', ascending=False)
# TODO choose the top weights coons, then the correlations with the becnchmark is high, change to sample num
# randomly
cons_ids = df['consID']
mkt_df = DataAPI.MktEqudGet(secID=cons_ids,
beginDate=start_date,
endDate=end_date,
isOpen=1,
pandas="1")
df_agg = mkt_df.groupby('secID').agg({'turnoverRate': ['mean']})
flatten_columns = [
'{0}_{1}'.format(item[0], item[1]) for item in df_agg.columns
]
df_agg.columns = flatten_columns
df_agg = df_agg.reset_index()
df_agg.sort_values(by='turnoverRate_mean', ascending=False)
sec_ids = df_agg['secID'][:_num]
ret.update({
'{0}.{1}'.format(_ticker, ticker_to_exchangecd.get(_ticker)):
sec_ids
})
return ret
def get_mkt_equd(self, security_ids=(), fields=None, start_date=None, end_date=None, asset_type='stock'):
ret = DataAPI.MktEqudGet(secID='603612.XSHG', field='closePrice', beginDate="20181103", endDate="20181130")
return ret
