def read_all_variables(self):
"""
Read all variables from RV estimate function
:return: RV, J, TJ, C, TC, LogReturn
"""
sample_num = 288
all_RVs, estimation, self.logreturn = CAL_AllRVs(coin=self.coin,
freq=self.freq,
sample_num=sample_num,
refresh=self.refresh,
refresh_est=self.refresh_est,
cv=self.cv,
truncate_zero=self.truncate_zero,
alpha=self.alpha,
annualized=self.annualized,
tz_lag=self.tz_lag)
self.rv = all_RVs['RV']
self.jump = all_RVs[f'Jump_{self.alpha}']
self.jump[self.jump < 0] = 0
self.t_jump = all_RVs[f'CTJump_{self.alpha}']
self.t_jump[self.t_jump < 0] = 0
self.c = self.rv - self.jump
self.tc = self.rv - self.t_jump
return self.rv, self.jump, self.t_jump, self.c, self.tc, self.logreturn
def jump_summary_statistics(coin,
freq,
cv=3,
refresh=True,
backup_data=True,
alpha=0.99,
annualized=True):
# TODO: Jumps dynamics with different alphas and cvs for different coins
# coin = 'gemini_BTC'
# freq = '5min'
# cv = 3
# refresh = True
# backup_data = True
# alpha = 0.99
alphas = [0.5, 0.95, 0.99, 0.999, 0.9999]
cvs = [1, 2, 2.5, 3, 3.5, 4, 5]
stats_out_dir = outdata_dir + 'StatisticsResults/'
os.makedirs(stats_out_dir, exist_ok=True)
all_RVs, estimation, logreturn = CAL_AllRVs(coin=coin,
freq=freq,
refresh=refresh,
refresh_est=False,
backup_data=backup_data,
cv=cv,
truncate_zero=False,
alpha=alpha,
annualized=annualized)
def main_test_func():
# ======Test Code for class object
# coins = ['gemini_BTC', 'gemini_ETH']
#
# freqs = ['5min']
# cvs = [3]
# alpha = 0.9999
# sample_num = 288
# tz_lag = 0
# For test special case
# logreturn_hf, default = read_coin_diff_freq(coin)
# coin = coins[0]
# freq = freqs[0]
# cv = cvs[0]
# alpha = alphas[0]
params = itertools.product(coins, freqs, alphas, cvs)
for coin, freq, alpha, cv in params:
print(f'{coin} at {freq}, cv={cv}, alpha={alpha}')
try:
# ! REFRESH ALL ESTIMATION RESULTS
all_RVs, estimation, logreturn = CAL_AllRVs(coin=coin,
freq=freq,
cv=cv,
refresh=True,
sample_num=sample_num,
refresh_est=True,
truncate_zero=True,
alpha=alpha,
annualized=True,
tz_lag=tz_lag)
except Exception as e:
print(e)
continue
def plot_funcs(**kwargs):
# logreturn_hf, default = read_coin_diff_freq(coin)
# coin = coins[0]
# freq = freqs[0]
# cv = cvs[0]
params = itertools.product(coins, freqs, cvs)
for coin, freq, cv in params:
print(f'{coin} in {freq}, cv={cv}')
all_RVs, estimation, logreturn = CAL_AllRVs(coin,
freq,
refresh=True,
refresh_est=False,
sample_num=sample_num,
cv=cv,
truncate_zero=True,
alpha=alpha,
annualized=True,
tz_lag=tz_lag)
if 'rvs_jump' in kwargs:
# Plot RV, BPV and jump separation on specified Cryptos
plot_crypto_jumpseparation(coin=coin,
freq=freq,
all_RVs=all_RVs,
logreturn=logreturn,
cv=cv,
alpha=alpha)
if 'kde' in kwargs:
# Plot KDE
plot_crypto_rvs_kde(all_RVs=all_RVs,
logreturn=logreturn,
coin=coin,
freq=freq,
cv=cv,
bin_num=100,
bandwidth=1.5)
if 'kde_logreturn' in kwargs:
# Plot KDE on logreturn
plt_logretrun_distribution(logreturn=logreturn,
asset_name=coin,
freq=freq,
drop_zero=True)
if 'program_trading' in kwargs:
plot_program_trading()
def plot_crypto_kde(coin, freq, cv):
all_RVs, estimation, logreturn = CAL_AllRVs(coin=coin,
freq=freq,
refresh=True,
refresh_est=False,
cv=cv,
truncate_zero=False,
alpha=0.99,
annualized=True,
tz_lag=0)
all_RVs.dropna(axis=0, inplace=True)
# KDE on BPV
rv = all_RVs['RV']
bpv = all_RVs['BPV']
tbpv = all_RVs['CTBPV']
# jump_sig = all_RVs['Jump_0.99']
# tjump_sig = all_RVs['CTJump_0.99']
plot_kde(ts=rv,
rv_name='RV',
coin=coin,
freq=freq,
cv=cv,
bd_method='silverman')
plot_kde(ts=bpv,
rv_name='BPV',
coin=coin,
freq=freq,
cv=cv,
bd_method='silverman')
plot_kde(ts=tbpv,
rv_name='TBPV',
coin=coin,
freq=freq,
cv=cv,
bd_method='silverman')
def crypto_summarystats(coin='gemini_BTC',
freq='5min',
cv=3,
refresh=True,
alpha=0.9999,
decimal=2,
sample_num=288,
format_threshold=0.0001,
annualized=True,
truncate_zero=True):
"""
:param coin:
:param freq:
:param cv:
:param refresh:
:param backup_data:
:param alpha:
:return:
"""
# coin = 'gemini_BTC'
# freq = '5min'
# cv = 3
# refresh = True
# alpha = 0.9999
# annualized=True
# truncate_zero=True
# decimal = 2
"""
'RV', 'BPV', 'TPV', 'z', 'Jump_raw', 'Jump_0.99', 'CTBPV', 'CTTPV',
'ctz', 'CTJump_raw', 'CTJump_0.99'
"""
# Save dir
stat_out_dir = outdata_dir + f'SummaryStats/{coin}_{freq}/'
os.makedirs(stat_out_dir, exist_ok=True)
# Calculate RVs estimators
all_RVs, estimation, logreturn = CAL_AllRVs(coin=coin,
freq=freq,
refresh=refresh,
refresh_est=False,
cv=cv,
sample_num=sample_num,
truncate_zero=truncate_zero,
alpha=alpha,
annualized=annualized,
tz_lag=0)
# Preprocessing
all_RVs.dropna(axis=0, inplace=True)
# Add More Variables
all_RVs['RV_sqrt'] = np.sqrt(all_RVs['RV'])
all_RVs['log(RV)'] = np.log(all_RVs['RV'])
all_RVs['C'] = all_RVs['RV'] - all_RVs[f'Jump_{alpha}']
all_RVs.loc[all_RVs['C'] == 0, 'C'] = np.nan
all_RVs['log(C)'] = np.log(all_RVs['C'])
all_RVs['TC'] = all_RVs['RV'] - all_RVs[f'CTJump_{alpha}']
all_RVs.loc[all_RVs['TC'] == 0, 'TC'] = np.nan
all_RVs['log(TC)'] = np.log(all_RVs['TC'])
# all_RVs['J_sqrt'] = np.sqrt(all_RVs[f'Jump_{alpha}'])
all_RVs['log(J+1)'] = np.log(all_RVs[f'Jump_{alpha}'] + 1)
# all_RVs.loc[all_RVs[f'Jump_{alpha}'] == 0, f'Jump_{alpha}'] = np.nan
# all_RVs['TJ_sqrt'] = np.sqrt(all_RVs[f'CTJump_{alpha}'])
all_RVs['log(TJ+1)'] = np.log(all_RVs[f'CTJump_{alpha}'] + 1)
# all_RVs.loc[all_RVs[f'CTJump_{alpha}'] == 0, f'CTJump_{alpha}'] = np.nan
drop_zero_col = [f'Jump_{alpha}', f'CTJump_{alpha}']
summary_stats_1coin = SummaryStatistics(estimator_matrix=all_RVs,
drop_zero_col=drop_zero_col,
lower_percentile=0.05,
higher_percentile=0.95,
acf_lags=[1, 7, 30, 100])
# Output statistics
main_vars = [
'RV', 'log(RV)', 'C', 'log(C)', 'TC', 'log(TC)', f'Jump_{alpha}',
'log(J+1)', f'CTJump_{alpha}', 'log(TJ+1)'
]
# main_vars = ['RV', 'RV_sqrt', 'log(RV)', 'C', 'C_sqrt', 'log(C)', 'TC', 'TC_sqrt', 'log(TC)', f'Jump_{alpha}',
# 'J_sqrt', 'log(J+1)', f'CTJump_{alpha}', 'TJ_sqrt', 'log(TJ+1)']
auxi_vars = (summary_stats_1coin.columns) ^ (main_vars)
main_var_statistics = summary_stats_1coin[main_vars]
auxilinary_statistics = summary_stats_1coin[auxi_vars]
for var in main_vars:
main_var_statistics[var] = reformat_stats_number(
main_var_statistics[var],
decimal_round=decimal,
format_threshold=format_threshold)
for var in auxi_vars:
auxilinary_statistics[var] = reformat_stats_number(
auxilinary_statistics[var],
decimal_round=decimal,
format_threshold=format_threshold)
# main_var_statistics.columns = ['RV1/2', 'log(RV)', 'C', 'TC', f'Jump(alpha)', f'TJump(alpha)']
main_var_statistics.to_latex(
stat_out_dir +
f'Main_{coin}_{freq}_{alpha}_{cv}_{decimal}_{format_threshold}.csv')
auxilinary_statistics.to_latex(
stat_out_dir +
f'Aux_{coin}_{freq}_{alpha}_{cv}_{decimal}_{format_threshold}.csv')
# print(main_var_statistics.to_latex(f'test.csv'))
# print(auxilinary_statistics.to_latex())
return main_var_statistics, auxilinary_statistics
def plot_crypto_jumpseparation(coin='gemini_BTC',
freq='5min',
cv=3,
alpha=0.99,
refresh=True):
# coin = 'gemini_BTC'
# freq = '5min'
# cv = 3
# alpha = 0.99
# refresh = True
all_RVs, estimation, logreturn = CAL_AllRVs(coin=coin,
freq=freq,
refresh=refresh,
sample_num=288,
refresh_est=True,
cv=cv,
truncate_zero=False,
alpha=alpha,
annualized=False,
tz_lag=0)
"""
'RV', 'BPV', 'TPV', 'z', 'Jump_raw', 'Jump_sig', 'CTBPV', 'CTTPV', 'ctz', 'CTJump_raw', 'CTJump_sig'
"""
rv = all_RVs['RV']
bpv = all_RVs['BPV']
ctpv = all_RVs['CTBPV']
jump_raw = all_RVs['Jump_raw']
jump_sig = all_RVs[f'Jump_{alpha}']
tjump = all_RVs['CTJump_raw']
tjump_sig = all_RVs[f'CTJump_{alpha}']
plotter = RealizedVolatilityPlot_OneAsset(return_ts=logreturn,
rv=rv,
bpv=bpv,
asset_name=coin,
ts_freq=freq)
rv_plot_dir = outplot_dir + f'RV_Separation/{coin}/'
os.makedirs(rv_plot_dir, exist_ok=True)
# Plot threshold RV separation
# Raw Jump from BPV
# jump = rv - bpv
rvs_fig = plotter.plt_threshold_realized_jumps(bpv=bpv,
jump=jump_raw,
title_bpv='$BPV^{1/2}$',
title_jump='$J^{1/2}$')
rvs_fig.savefig(f'{rv_plot_dir}{coin}_{freq}_BPVRawJumps_{cv}.png',
dpi=300)
plt.close()
# BPV significant Jumps
rvs_sig_fig = plotter.plt_threshold_realized_jumps(
bpv=bpv,
jump=jump_sig,
title_bpv='$BPV^{1/2}$',
title_jump='$J^{1/2},$' + r'$\alpha$' + f"$={alpha}$")
rvs_sig_fig.savefig(
f'{rv_plot_dir}{coin}_{freq}_BPVSignificantJumps_{cv}_{alpha}.png',
dpi=300)
plt.close()
# CTBPV raw Jumps
ctrvs_fig = plotter.plt_threshold_realized_jumps(
bpv=ctpv,
jump=tjump,
title_bpv='$TBPV^{1/2},$' + r'$C_\theta$' + f"={cv}",
title_jump='$TJ^{1/2}$')
ctrvs_fig.savefig(f'{rv_plot_dir}{coin}_{freq}_CTRawJumps_{cv}.png',
dpi=300)
plt.close()
# CTBPV significant Jumps
ctrvs_sig_fig = plotter.plt_threshold_realized_jumps(
bpv=ctpv,
jump=tjump_sig,
title_bpv='$TBPV^{1/2},$'
r'$C_\theta$' + f"={cv}",
title_jump='$TJ^{1/2},$' + r'$\alpha$' + f"$={alpha}$")
ctrvs_sig_fig.savefig(
f'{rv_plot_dir}{coin}_{freq}_CTBPVSignificantJumps_{cv}_{alpha}.png',
dpi=300)
plt.close()
plot_crypto_kde(coin='gemini_ETH', freq='5min', cv=3)
plot_crypto_kde(coin='ETH', freq='5min', cv=3)
coins = ['gemini_BTC', 'gemini_ETH']
freqs = ['5min', '10min', 'min']
cvs = [1, 2, 3, 4, 5, 6, 7, 8, 9]
# logreturn_hf, default = read_coin_diff_freq(coin)
coin = coins[0]
freq = freqs[0]
cv = 3
sample_num = 288
all_RVs, estimation, logreturn = CAL_AllRVs(coin,
freq,
refresh=True,
refresh_est=False,
sample_num=sample_num,
cv=cv,
truncate_zero=True,
alpha=0.99,
annualized=True,
tz_lag=0)
# plot daily price, rv, big significant jumps
sp_date = dt.date(2017, 3, 10)
coin_data = read_clean_gemini(coin, freq, True)
tj_sorted = all_RVs[f'CTJump_0.99'].sort_values(ascending=False)
high_tj_dates = tj_sorted.head(10).index
close = coin_data['Close']
close_daily = close.groupby(by=close.index.date,
axis=0).apply(lambda x: x.tail(1).sum())
def construct_variables_over_cryptos(
freq,
alpha,
cv,
annualized=True,
):
"""
Construct each variable over different asset
:return:
"""
stats_out_dir = outdata_dir + f'Vars_AllCryptos/'
os.makedirs(stats_out_dir, exist_ok=True)
assets = ['gemini_BTC', 'gemini_ETH', 'BTC', 'ETH', 'XRP', 'LTC']
# all params
refresh = True
truncate_zero = True
# annualized = True
#
# freq = freqs[0]
# alpha = alphas[0]
# cv = cvs[1]
# Extract RV variable over all cryptos
rvs_file = outdata_dir + f'Vars_AllCryptos/RV_{freq}_{alpha}_{cv}.csv'
zs_file = outdata_dir + f'Vars_AllCryptos/z-stat_{freq}_{alpha}_{cv}.csv'
tzs_file = outdata_dir + f'Vars_AllCryptos/tz-sta_{freq}_{alpha}_{cv}.csv'
js_file = outdata_dir + f'Vars_AllCryptos/Jump_Sig_{freq}_{alpha}_{cv}.csv'
tjs_file = outdata_dir + f'Vars_AllCryptos/tJump_sig_{freq}_{alpha}_{cv}.csv'
exist_flag = all([
os.path.exists(file_name)
for file_name in [rvs_file, zs_file, tzs_file, js_file, tjs_file]
])
if exist_flag:
rvs = pd.read_csv(rvs_file, index_col=0, parse_dates=True)
zs = pd.read_csv(zs_file, index_col=0, parse_dates=True)
tzs = pd.read_csv(tzs_file, index_col=0, parse_dates=True)
js = pd.read_csv(js_file, index_col=0, parse_dates=True)
tjs = pd.read_csv(tjs_file, index_col=0, parse_dates=True)
else:
rvs = pd.DataFrame(columns=assets)
zs = pd.DataFrame(columns=assets)
tzs = pd.DataFrame(columns=assets)
js = pd.DataFrame(columns=assets)
tjs = pd.DataFrame(columns=assets)
for coin in rvs.keys():
print(coin)
all_RVs, estimation, logreturn = CAL_AllRVs(
coin=coin,
freq=freq,
refresh=refresh,
refresh_est=False,
cv=cv,
truncate_zero=truncate_zero,
alpha=alpha,
annualized=annualized,
tz_lag=0)
rvs.loc[:, coin] = all_RVs['RV']
zs.loc[:, coin] = all_RVs['z']
tzs.loc[:, coin] = all_RVs['ctz']
js.loc[:, coin] = all_RVs[f'Jump_{alpha}']
tjs.loc[:, coin] = all_RVs[f'CTJump_{alpha}']
rvs.to_csv(outdata_dir + f'Vars_AllCryptos/RV_{freq}_{alpha}_{cv}.csv')
zs.to_csv(outdata_dir +
f'Vars_AllCryptos/z-stat_{freq}_{alpha}_{cv}.csv')
tzs.to_csv(outdata_dir +
f'Vars_AllCryptos/tz-sta_{freq}_{alpha}_{cv}.csv')
js.to_csv(outdata_dir +
f'Vars_AllCryptos/Jump_Sig_{freq}_{alpha}_{cv}.csv')
tjs.to_csv(outdata_dir +
f'Vars_AllCryptos/tJump_sig_{freq}_{alpha}_{cv}.csv')
log_rvs = rvs.replace(0, np.nan)
log_rvs = np.log(log_rvs)
log_js = np.log(js + 1)
log_tjs = np.log(tjs + 1)
cs = rvs - js
log_cs = cs.replace(0, np.nan)
log_cs = np.log(log_cs)
tcs = rvs - tjs
log_tcs = tcs.replace(0, np.nan)
log_tcs = np.log(log_tcs)
return rvs, log_rvs, js, log_js, tjs, log_tjs, cs, log_cs, tcs, log_tcs