def _get_comb():
eretM = load_data('eretM')
eretM = eretM.stack()
eretM.index.names = ['t', 'sid']
eretM.name = 'eret'
pu = load_data('pu')
combM = eretM.to_frame().join(pu)
return combM
def analyse_outliers(data):
#TODO: a little messy
detect_outliers(data, 'data')
#---------------------------liquidity----------------------------------
liquidity=load_data('liquidity')
#amihud
amihud=liquidity['amihud'].unstack()
amihud=delete_outliers(amihud,method='percentile',thresh=98,pooled=True)
detect_outliers(amihud, 'amihud')
#filter ps1
ps1=liquidity['ps1'].unstack()
ps1_pooled=delete_outliers(ps1,method='percentile',thresh=99,pooled=True)
detect_outliers(ps1_pooled, 'ps1_pooled')
#ps2
ps2=liquidity['ps2'].unstack()
ps2_pooled=delete_outliers(ps2,method='percentile',thresh=99,pooled=True)
detect_outliers(ps2_pooled, 'ps2_pooled')
#roll1
roll1=liquidity['roll1'].unstack()
roll1_pooled=delete_outliers(roll1,method='percentile',thresh=99,pooled=True)
detect_outliers(roll1_pooled, 'roll1_pooled')
#roll2
roll2=liquidity['roll2'].unstack()
roll2_pooled=delete_outliers(roll2,method='percentile',thresh=99,pooled=True)
detect_outliers(roll2_pooled, 'roll2_pooled')
#zeros1
zeros1=liquidity['zeros1'].unstack()
zeros1_pooled=delete_outliers(zeros1,method='percentile',thresh=99,pooled=True)
detect_outliers(zeros1_pooled, 'zeros1_pooled')
#zeros2
zeros2=liquidity['zeros2'].unstack()
zeros2_pooled=delete_outliers(zeros2,method='percentile',thresh=99,pooled=True)
detect_outliers(zeros2_pooled, 'zeros2_pooled')
#----------------------------skewness----------------------------------------
skewness=load_data('skewness')
idioskew_24M__D=skewness['idioskew_24M__D'].unstack()
idioskew_24M__D_pooled=delete_outliers(idioskew_24M__D,method='percentile',thresh=99.9,pooled=True)
detect_outliers(idioskew_24M__D_pooled,'idioskew_24M__D_pooled')
skew_12M__D=skewness['skew_12M__D'].unstack()
skew_12M__D_pooled=delete_outliers(skew_12M__D,method='percentile',thresh=99.9,pooled=True)
detect_outliers(skew_12M__D_pooled,'skew_12M__D_pooled')
skew_24M__D=skewness['skew_24M__D'].unstack()
skew_24M__D_pooled=delete_outliers(skew_24M__D,method='percentile',thresh=99.9,pooled=True)
detect_outliers(skew_24M__D_pooled,'skew_24M__D_pooled')
def get_momentum():
stockRetM=load_data('stockRetM')
stk=stockRetM.stack()
stk.index.names=['t','sid']
#lagged 1 month
#Te one month lag is imposed to avoid the short-term reversal effect frst documented by Jegadeesh (1990)
d_lag=OrderedDict({'mom':[12,9],#since the window is 11,and we do not use the value of time t,so,here we set 12 rather than 11
'r12':[13,10],
'r6':[7,5]})
#nonlagged
d_nonlag=OrderedDict({'R12M':[12,10],
'R9M':[9,7],
'R6M':[6,5],
'R3M':[3,3]})
ss=[]
names=[]
for bn,bp in d_lag.items():
ser=stk.groupby('sid').apply(lambda s:_before(s,bp[0],bp[1]))
ss.append(ser)
names.append(bn)
for un,up in d_nonlag.items():
ser=stk.groupby('sid').apply(lambda s:_upto(s,up[0],up[1]))
ss.append(ser)
names.append(un)
momentum=pd.concat(ss,axis=1,keys=names)
momentum.columns.name='type'
momentum=momentum*100
momentum.columns.name='type'
save(momentum,'momentum',sort_axis=False)
def get_predicted(history):
params = pd.read_csv(fn, index_col=0, parse_dates=True)
params = params.rolling(window=history, min_periods=int(
history / 2)).mean() #TODO:min_periods
# we will use the parameters of time t to predict the
# returns in time t,so shift forward params for 1 step.
params = params.shift(1)
indicators = load_data('data')[l]
indicators['Intercept'] = 1.0
cols = params.columns
indicators = indicators.reindex(columns=cols)
#TODO: predict return rather than eret,
groups = list(indicators.groupby('sid'))
ss = []
names = []
for name, g in groups:
g = g.reset_index(level='sid', drop=True)
p, g = get_inter_frame([params, g.dropna()])
s = (p * g).sum(axis=1)
ss.append(s)
names.append(name)
print(name)
predicted = pd.concat(ss, axis=1, keys=names)
predicted.to_pickle(
os.path.join(directory, 'predicted_{}.pkl'.format(history)))
def handle_data():
size = load_data('size')['size']
save(size, 'size')
price = load_data('stockCloseM').stack()
price.name = 'price'
save(price, 'price')
beta = pd.read_csv(os.path.join(PATH, 'beta.csv'),
index_col=[0, 1],
parse_dates=True)['beta']
save(beta, 'beta')
sd = pd.read_csv(os.path.join(PATH, 'sd.csv'),
index_col=[0, 1],
parse_dates=True)['sd']
save(sd, 'sd')
see = pd.read_csv(os.path.join(PATH, 'see.csv'),
index_col=[0, 1],
parse_dates=True)['see']
save(see, 'see')
strev = pd.read_csv(os.path.join(PATH, 'strev.csv'),
index_col=[0, 1],
parse_dates=True)['strev']
save(strev, 'strev')
factors = pd.read_csv(os.path.join(PATH, 'factors.csv'),
index_col=[0, 1],
parse_dates=True)
factors.head()
fns = list(filter(lambda fn: fn.endswith('.csv'), os.listdir(PATH)))
dfs = []
for fn in fns:
df = pd.read_csv(os.path.join(PATH, fn),
index_col=[0, 1],
parse_dates=True)
dfs.append(df)
print(fn, df.shape)
comb = pd.concat(dfs, axis=1)
comb = comb.dropna(axis=0, how='all')
save(comb, 'comb')
def get_hxz4():
'''
calculate hxz4 factors,refer to din.py for details about the indicators
References:
Hou, K., Mo, H., Xue, C., and Zhang, L. (2018). Motivating Factors (Rochester, NY: Social Science Research Network).
Returns:
'''
v1 = 'size__size'
v2 = 'inv__inv' #I/A
v3 = 'roe__roe' # ROE
comb = combine_with_datalagged([v1, v2, v3], sample_control=True)
comb = comb.dropna()
comb['g1'] = comb.groupby('t', group_keys=False).apply(
lambda df: assign_port_id(df[v1], 2, range(1, 3)))
comb['g2'] = comb.groupby(['t', 'g1'], group_keys=False).apply(
lambda df: assign_port_id(df[v2], [0, 0.3, 0.7, 1.0], range(1, 4)))
comb['g3'] = comb.groupby(['t', 'g1'], group_keys=False).apply(
lambda df: assign_port_id(df[v3], [0, 0.3, 0.7, 1.0], range(1, 4)))
assets = comb.groupby(
['t', 'g1', 'g2',
'g3']).apply(lambda df: my_average(df, 'stockEretM', wname='weight'))
df1 = assets.groupby(['t', 'g1']).mean().unstack(level='g1')
smb = df1[1] - df1[2]
df2 = assets.groupby(['t', 'g2']).mean().unstack(level='g2')
ria = df2[3] - df2[1]
df3 = assets.groupby(['t', 'g3']).mean().unstack(level='g3')
roe = df3[3] - df2[1]
rp = load_data('rpM')
hxz4 = pd.concat([rp, smb, ria, roe],
axis=1,
keys=['rp', 'smb', 'ria', 'roe'])
hxz4.columns.name = 'type'
hxz4 = hxz4.dropna()
save(hxz4, 'hxz4M')
def lagged_n(n=0):
predicted = pd.read_pickle(
os.path.join(directory, 'predicted_{}.pkl'.format(history)))
eret = load_data('stockEretM').shift(-n).stack()
comb = pd.concat([eret, predicted.stack().replace(0, np.nan)],
axis=1,
keys=['eret', 'predicted'])
comb.index.names = ['t', 'sid']
comb = comb.dropna()
comb['g'] = comb.groupby(
't', group_keys=False).apply(lambda df: assign_port_id(
df['predicted'], 10, ['g{}'.format(i) for i in range(1, 11)]))
ts = comb.groupby(['t', 'g'])['eret'].mean().unstack('g')
ts.columns = ts.columns.astype(str)
ts['spread'] = ts['g10'] - ts['g1']
print(n)
return ts['spread']
def regress_predicted_on_realized():
predicted = pd.read_pickle(
os.path.join(directory, 'predicted_{}.pkl'.format(history)))
stockEret = load_data('stockEretM')
predicted, stockEret = get_inter_frame([predicted, stockEret])
months = []
models = []
count = []
for month, p in predicted.iterrows():
# p denotes predicted return
# r denotes realized return
r = stockEret.loc[month]
df = pd.concat([p, r], axis=1, keys=['predicted', 'realized'])
df = df.dropna()
model = sm.ols(formula='realized ~ predicted', data=df).fit(use_t=True)
months.append(month)
models.append(model)
count.append(df.shape[0])
print(month)
slope = pd.Series([m.params['predicted'] for m in models], index=months)
r2 = pd.Series([m.rsquared for m in models], index=months)
n = pd.Series(count, index=months)
plt.plot(slope.index, slope.values, 'o')
plt.show()
plt.plot(r2.index, r2.values, 'o')
plt.show()
plt.plot(n.index, n.values)
plt.show()
slope.max().max()
slope.min().min()
slope.describe()
r2.describe()
nw = newey_west(formula='predicted ~ 1',
df=pd.DataFrame(slope, columns=['predicted']),
lags=5)
def visulize_pu():
pu = load_data('pu')
pu.plot()
plt.show()