def binningHFseries(time_trades,flag_dt,dx=None):
# This function computes the number of trades and the traded volume at
# time intervals of 1-second
# INPUT
# time_trades :[row vector] trade times
# flag_dt :[string] to select the frequency of observations
# dx :[row vector] traded volume at time_of_trades. It is optional. Input dx only if interested in dv
# OP
# dk :[row vector] number of trades between t and t+dt
# k :[row vector] cumulative number of trades at time t
# t :[row vector] time binned with frequency dt
# dq :[row vector] traded volume between t and t+dt.
## Code
if dx is None:
dx = zeros((1,len(time_trades)))
# vector of time
time_delta = date_mtop(npmax(time_trades))-date_mtop(npmin(time_trades))
if flag_dt=='1second':
T = time_delta.total_seconds() # delta_t = 1 second
# delta_t = (npmax(time_trades)-npmin(time_trades))/T
# t = arange(npmin(time_trades),npmax(time_trades), delta_t)
t = linspace(npmin(time_trades), npmax(time_trades), int(T)+1)
#compute the number of events for each delta_t
dk, dq = zeros((1, len(t)-1)), zeros((1, len(t)-1))
for i in range(len(t)-1):
if i==len(t)-1:
dk[0,i] = np.sum((time_trades >= t[i]) & ( time_trades<=t[i+1] ) )
dq[0,i] = np.sum(dx[0, (time_trades >= t[i])&(time_trades<=t[i+1])])
else:
dk[0,i] = np.sum((time_trades >= t[i]) & ( time_trades<t[i+1] ) )
dq[0,i] = np.sum(dx[0, (time_trades >= t[i])&(time_trades<t[i+1])])
k = cumsum(dk) #series over delta_t intervals
return dk, k, t, dq
t_k = trades.time # time vector of trades
p_last = trades.price # last transaction prices
delta_q = trades.siz # flow of traded contracts' volumes
delta_sgn = trades.aggress # trade sign flow
match = trades.mtch # match events: - the "1" value indicates the "start of a match event" while zeros indicates the "continuation of a match event"
# - the db is ordered such that the start of a match event is in the last column corresponding to that event
t, _, _, _, _, _, t_k, _, _, _, _, _ = TradeQuoteProcessing(t, dates_quotes, q_ask, p_ask, q_bid, p_bid, t_k, dates_trades,
p_last, delta_q, delta_sgn, match)
t = t.flatten()
t_k = t_k.flatten()
# ## Compute the gaps between subsequent events
k_0 = where(t_k >= t[0])[0][0] # index of the first trade within the time window
k_1 = where(t_k <= t[-1])[0][-1] # index of the last trade within the time window
ms = (date_mtop(t_k[k_1]) - date_mtop(t_k[k_0])).seconds * 1000 + (date_mtop(t_k[k_1]) - date_mtop(t_k[k_0])).microseconds / 1000
t_k = linspace(t_k[k_0],t_k[k_1], int(ms)) # time window's wall-clock-time vector expressed in milliseconds
delta_t_k = diff(t_k) # waiting times
# -
# ## ACD fit (Requires the external package ACD_Models_FEX)
# +
q = 1 # maximum lag for the duration
p = 1 # maximum lag for the volatility
stdMethod = 1
tmp_dt_n = [0, delta_t_k]
specOut = ACD_Fit(tmp_dt_n.T,'exp', q, p, stdMethod) # fitting
# estimated parameters
c = specOut.w
opt_p = exp(weights[0] * log(p1) + weights[1] * log(p2))
opt_p = opt_p / npsum(opt_p)
ens_optp = EffectiveScenarios(opt_p, typ)
# ## S&P returns histogram and statistics
option = namedtuple('option', 'n_bins')
option.n_bins = 10 * log(epsi.shape[1])
p_eps, x_eps = HistogramFP(epsi, opt_p, option)
m, Sdev, VaR, CVaR, Sk, K = Stats(epsi, opt_p)
# ## Generate the figure
# +
date_tick = arange(99, len(date), 380)
date_dt = array([date_mtop(i) for i in date])
myFmt = mdates.DateFormatter('%d-%b-%y')
figure(figsize=(16, 10))
# VIX
ax = plt.subplot2grid((2, 5), (0, 0), colspan=2)
ph0 = ax.plot(date_dt, p1[0], lw=0.5, color='gray')
xticks([])
yticks([])
ax2 = ax.twinx()
ax2.plot(date_dt, z1[0], color=[0, 0, 0.6], lw=0.5)
ph1 = ax2.plot(date_dt, z1[0, -1] * ones(t_), color='r', linestyle='--')
xlim([min(date_dt), max(date_dt)])
ax.set_ylim([0, 1.5 * npmax(p1)])
ax2.set_ylim([npmin(z1), 1.3 * npmax(z1)])
ax2.set_yticks(arange(20, 100, 20))
db = loadmat(os.path.join(TEMPORARY_DB, 'db_Stocks'), squeeze_me=True) SPX = struct_to_dict(db['SPX']) # daily prices and log-prices price = SPX.Price_close date = SPX.Date logprice = log(price) # - # ## Compute weekly prices, returns and the log-square volatility # ##pick weekly data # + w = arange(0, len(logprice), 5) date = array([date_mtop(i) for i in SPX.Date[w]]) # prices price = price[w] # log-prices logprice_weekly = log(price) # log-returns ret = diff(logprice_weekly) # y = log(squared returns) y = log(ret**2) # - # ## Fit the stochastic volatility model
option = namedtuple('option', 'n_bins')
option.n_bins = round(10 * log(t_))
p, x = HistogramFP(pnl.reshape(1, -1), flex_probs, option)
# ## Compute the MMFP pdf
# +
xx = sort(x)
xx = r_[xx, npmax(xx) + arange(0.001, 0.051, 0.001)]
m1 = flex_probs @ pnl.T
m3 = flex_probs @ ((pnl - m1)**3).T
sln = lognorm.pdf(sign(m3) * xx - c, sqrt(sig2), scale=exp(mu)) # fitted pdf
date_dt = array([date_mtop(datenum(i)) for i in date])
myFmt = mdates.DateFormatter('%d-%b-%Y')
date_tick = arange(200 - 1, t_, 820)
# -
# ## Generate the figure
# +
f = figure()
# HFP histogram with MMFP pdf superimposed
h1 = plt.subplot(3, 1, 1)
b = bar(x[:-1],
p[0],
width=x[1] - x[0],
facecolor=[.8, .8, .8],
edgecolor=[.6, .6, .6])
t, _, q_ask, p_ask, q_bid, p_bid, t_k, _, p_last, delta_q, delta_sgn, vargout = \
TradeQuoteProcessing(t, dates_quotes, q_ask, p_ask, q_bid, p_bid, t_k, dates_trades, p_last, delta_q, delta_sgn, match,
{1:numords_ask, 2:numords_bid})
t = t.flatten()
numords_ask, numords_bid = vargout[1], vargout[2]
q = cumsum(delta_q) # cumulative volume of traded contracts
sgn = cumsum(delta_sgn) # cumulative trade sign
# -
# ## Compute the time series of the cumulative volume and the cumulative sign as functions of wall clock time using function TradeQuoteSpreading
# +
i_t0 = 1 # index of window's starting time
i_t1 = 1250 # index of window's last time
ms = (date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])).seconds * 1000 + (
date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])).microseconds / 1000 + 1
# total len of time window expressed in wall-clock-time
t_ms = linspace(
t[i_t0 - 1], t[i_t1 - 1],
int(ms) +
1) # time window's wall-clock-time vector expressed in milliseconds
k_0 = where(
t_k[0] >= t[i_t0])[0][0] # index of the first trade within the time window
k_1 = where(
t_k[0] <= t[i_t1])[0][-1] # index of the last trade within the time window
_, _, _, _, p_last, q, sgn, _ = TradeQuoteSpreading(
t_ms, t[i_t0 - 1:i_t1], q_ask[0, i_t0 - 1:i_t1], p_ask[0, i_t0 - 1:i_t1],
q_bid[0, i_t0 - 1:i_t1], p_bid[0, i_t0 - 1:i_t1], t_k[0, k_0:k_1 + 1],
# +
try:
db = loadmat(os.path.join(GLOBAL_DB, 'db_US_10yr_Future_quotes_and_trades'), squeeze_me=True)
except FileNotFoundError:
db = loadmat(os.path.join(TEMPORARY_DB, 'db_US_10yr_Future_quotes_and_trades'), squeeze_me=True)
trades = struct_to_dict(db['trades'])
# -
# ## Comupte the invariants: eps= dn = number of trades in 1-second time intervals
# +
flag_dt = '1second'
trade = unique(trades.time) # trade time
dk, k,*_ = binningHFseries(trade, flag_dt)
time = array([date_mtop(i) for i in arange(trade[0], trade[-1], 1.1574e-05)])
# time = cellstr((time))
time = time[1:]
epsi = dk
t_ = epsi.shape[1]
# -
# ## Set a Flexible Probabilities scenario using BlowSpinFP
U = norm.rvs(mean(epsi), 1, size=(1, t_))
b = 1
s = 0
FP1, ens1 = BlowSpinFP(r_[epsi, U], b, s, spinscale=0.2, method='direct')
flex_probs = FP1[[b-1 + s],:]
ens = ens1[b-1 + s]
FP = (1 / (t + 1)) * ones(t + 1)
y1_t = y1_l0[:t + 1]
y2_t = y2_l0[:t + 1]
FPstd1 = sqrt(npsum(FP * (y1_t**2)))
FPstd2 = sqrt(npsum(FP * (y2_t**2)))
rho2_l0[0, t] = npsum(FP * y1_t * y2_t) / (FPstd1 * FPstd2)
# -
# ## Generate the figures
# +
ln_p1 = logprice1[l + 1:]
ln_p2 = logprice2[l + 1:]
date = Date[l + 1:]
date_dt = array([date_mtop(i) for i in date])
myFmt = mdates.DateFormatter('%d-%b-%Y')
date_tick = date_dt[arange(99, len(date_dt), 200)]
# FIGURE 1: overlap 5-days (l=4)
f, ax = subplots(3, 1)
# correlation
plt.sca(ax[0])
plot(date_dt, rho2[0], color='k')
xlim([min(date_dt), max(date_dt)])
xticks(date_tick)
ylim([0.1, 1])
ax[0].xaxis.set_major_formatter(myFmt)
title('Correlation')
lag = 'overlap: % 2.0f days' % (l + 1)
# +
figure()
# scatter plot
scatter(100 * epsi_MC[0], epsi_MC[1], s=10, c=[0.6, 0.6, 0.6], marker='*')
title('COPULA-MARGINAL Distribution')
xlabel('S&P 500 daily return (%)')
ylabel('NS first parameter')
plt.axis([
100 * npmin(epsi_MC[0]) - 0.01, 100 * npmax(epsi_MC[0]) + 0.01,
npmin(epsi_MC[1]) - 0.01,
npmax(epsi_MC[1]) + 0.01
])
# vix plot
date_xtick = arange(99, len(vix), 380)
dates_dt = array([date_mtop(i) for i in dates])
xticklab = dates_dt[date_xtick]
myFmt = mdates.DateFormatter('%d-%b-%Y')
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])
figure()
ax1 = plt.gca()
plt.bar(dates_dt,
p[0],
width=dates_dt[1].toordinal() - dates_dt[0].toordinal(),
color='grey')
ax1.xaxis.set_major_formatter(myFmt)
ax2 = ax1.twinx()
ax2.plot(dates_dt, vix, color=[0, 0, 0.6], lw=1)
ax2.plot(dates_dt,
conditioner.TargetValue[0] * ones(t_),
except FileNotFoundError:
db = loadmat(os.path.join(TEMPORARY_DB, 'db_signals_size'), squeeze_me=True)
v = db['v']
t_start = db['t_start']
t_scor = db['t_scor']
t_smoo = db['t_smoo']
dates = db['dates']
s_siz_rk = db['s_siz_rk']
# settings
n_, _ = v.shape
t_ = v.shape[1] # final date
nu_x = log(2) / 100 # half-life for HFP covariance matrix of compounded returns
nu_ic = log(2) / 40 # half-life information coefficient
dates = array([date_mtop(int(i)) for i in dates[t_start:t_]])
s = s_siz_rk[:, t_start - t_smoo - t_scor+1:]
tsize = t_-t_start
sigma2_pnl_shr = zeros((n_,n_,tsize))
sigma2_pnl = zeros((n_,n_,tsize))
sigma2_h = zeros((1,tsize))
beta = zeros((n_,tsize))
h_mv = zeros((n_,tsize))
market_pnl = zeros((n_,tsize))
pnl_contr = zeros((n_,tsize))
ic_hat = zeros((1,tsize))
x = zeros((21,tsize))
y = zeros(x.shape)
1: numords_ask,
2: numords_bid
})
t = t.flatten()
numords_ask, numords_bid = vargout[1], vargout[2]
q = cumsum(delta_q) # cumulative volume of traded contracts
sgn = cumsum(delta_sgn) # cumulative trade sign
# -
# ## Compute the realized time series of new events delta_k_t with time unit of one second
# +
i_t0 = 1 # index of window's starting time
i_t1 = len(t) # index of window's last time
ms = (date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])).seconds * 1000 + (
date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])).microseconds / 1000
# total len of time window expressed in wall-clock-time
t_ms = linspace(
t[i_t0 - 1], t[i_t1 - 1],
int(ms) +
1) # time window's wall-clock-time vector expressed in milliseconds
k_0 = where(
t_k[0] >= t[i_t0 -
1])[0][0] # index of the first trade within the time window
k_1 = where(
t_k[0] <= t[i_t1 -
1])[0][-1] # index of the last trade within the time window
_, _, _, _, p_last, *_ = TradeQuoteSpreading(
t, _, q_ask, p_ask, q_bid, p_bid, t_k, _, p_last, delta_q, delta_sgn, vargout = TradeQuoteProcessing(t,dates_quotes,
q_ask,p_ask,q_bid,p_bid,t_k,dates_trades,p_last,delta_q,delta_sgn,match,{1:numords_ask,2:numords_bid})
numords_ask, numords_bid = vargout[1], vargout[2]
t = t.flatten()
t_k = t_k.flatten()
q = cumsum(delta_q) # cumulative volume of traded contracts
sgn = cumsum(delta_sgn) # cumulative trade sign
# -
# ## Compute the realized time series of new events delta_k_t with time unit of one second
# +
i_t0 = 0 # index of window's starting time
i_t1 = len(t)-1 # index of window's last time
ms = (date_mtop(t[i_t1]) - date_mtop(t[i_t0])).seconds*1000 + (date_mtop(t[i_t1])-date_mtop(t[i_t0])).microseconds/1000 +1
# total len of time window expressed in wall-clock-time
t_ms = linspace(t[i_t0],t[i_t1],int(ms))
k_0 = where(t_k >= t[i_t0])[0][0] # index of the first trade within the time window
k_1 = where(t_k <= t[i_t1])[0][-1] # index of the last trade within the time window
# from numba import double, jit
#
# fastTradeQuoteSpreading = jit((double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:],double[:,:]),
# (double[:], double[:], double[:], double[:], double[:], double[:], double[:], double[:],double[:], double[:]))\
# (TradeQuoteSpreading)
_, _, _, _, p_last, *_ = TradeQuoteSpreading(t_ms, t[i_t0:i_t1], q_ask[0,i_t0:i_t1], p_ask[0,i_t0:i_t1], q_bid[0,i_t0:i_t1],
p_bid[0,i_t0:i_t1], t_k[k_0:k_1], p_last[0,k_0:k_1], q[k_0:k_1],
sgn[k_0:k_1])
# - the db is ordered such that the start of a match event is in the last column corresponding to that event
t, dates_quotes, q_ask, p_ask, q_bid, p_bid, t_k, dates_trades, p_last, delta_q, delta_sgn, vargout = \
TradeQuoteProcessing(t, dates_quotes, q_ask, p_ask, q_bid, p_bid, t_k, dates_trades, p_last, delta_q, delta_sgn, match,
{1:numords_ask, 2:numords_bid})
t = t.flatten()
numords_ask, numords_bid = vargout[1], vargout[2]
q = cumsum(delta_q) # cumulative volume of traded contracts
sgn = cumsum(delta_sgn) # cumulative trade sign
# -
# ## Compute microprice and total exchanged volume as functions of wall clock time and volume time
# +
i_t1 = len(t) # index of window's last time
ms = (date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])).seconds * 1000 + (
date_mtop(t[i_t1 - 1]) - date_mtop(t[i_t0 - 1])
).microseconds / 1000 # total len of time window expressed in wall-clock-time
t_ms = linspace(
t[i_t0 - 1], t[i_t1 - 1],
int(ms) +
1) # time window's wall-clock-time vector expressed in milliseconds
k_0 = where(
t_k[0] >= t[i_t0 -
1])[0][0] # index of the first trade within the time window
k_1 = where(
t_k[0] <= t[i_t1 -
1])[0][-1] # index of the last trade within the time window
q_ask, p_ask, q_bid, p_bid, _, q_t, _, _ = TradeQuoteSpreading(
t_end = array([
'03-Oct-2003', '03-Oct-2004', '03-Oct-2005', '03-Oct-2006', '03-Oct-2007'
])
_, timeindex, _ = intersect(time, list(map(datenum, t_end)))
# -
# ## Interpolate the rolling value on an yearly spaced grid
zroll = interp(time, dates, zroll[0])
# ## Plot the evolution of the rolling values highlighting them at times t = 03-Oct-2002,...,03-Oct-2006
# rolling value plot
figure()
time_dt = array([date_mtop(i) for i in time])
plot(time_dt, zroll, zorder=1)
scatter(time_dt[timeindex[:-1]],
zroll[timeindex[:-1]],
marker='.',
s=100,
c='r',
zorder=2)
plt.xticks(time_dt[timeindex])
myFmt = mdates.DateFormatter('%d-%b-%Y')
plt.gca().xaxis.set_major_formatter(myFmt)
xlim([time_dt[0], time_dt[timeindex[-1]]])
plt.grid(True)
title('Rolling prices')
# save_plot(ax=plt.gca(), extension='png', scriptname=os.path.basename('.')[:-3], count=plt.get_fignums()[-1])
