def dfm(X, Spec, threshold=1e-5, max_iter=5000):
# DFM() Runs the dynamic factor model
#
# Syntax:
# Res = DFM(X,Par)
#
# Description:
# DFM() inputs the organized and transformed data X and parameter structure Par.
# Then, the function outputs dynamic factor model structure Res and data
# summary statistics (mean and standard deviation).
#
# Input arguments:
# X: Kalman-smoothed data where missing values are replaced by their expectation
# Par: A structure containing the following parameters:
# Par.blocks: Block loadings.
# Par.nQ: Number of quarterly series
# Par.p: Number of lags in transition matrix
# Par.r: Number of common factors for each block
#
# Output Arguments:
#
# Res - structure of model results with the following fields
# . X_sm | Kalman-smoothed data where missing values are replaced by their expectation
# . Z | Smoothed states. Rows give time, and columns are organized according to Res.C.
# . C | Observation matrix. The rows correspond
# to each series, and the columns are organized as shown below:
# - 1-20: These columns give the factor loa dings. For example, 1-5
# give loadings for the first block and are organized in
# reverse-chronological order (f^G_t, f^G_t-1, f^G_t-2, f^G_t-3,
# f^G_t-4). Columns 6-10, 11-15, and 16-20 give loadings for
# the second, third, and fourth blocks respectively.
# .R: Covariance for observation matrix residuals
# .A: Transition matrix. This is a square matrix that follows the
# same organization scheme as Res.C's columns. Identity matrices are
# used to account for matching terms on the left and righthand side.
# For example, we place an I4 matrix to account for matching
# (f_t-1; f_t-2; f_t-3; f_t-4) terms.
# .Q: Covariance for transition equation residuals.
# .Mx: Series mean
# .Wx: Series standard deviation
# .Z_0: Initial value of state
# .V_0: Initial value of covariance matrix
# .r: Number of common factors for each block
# .p: Number of lags in transition equation
#
# References:
#
# Marta Banbura, Domenico Giannone and Lucrezia Reichlin
# Nowcasting (2010)
# Michael P. Clements and David F. Hendry, editors,
# Oxford Handbook on Economic Forecasting.
## Store model parameters ------------------------------------------------
# DFM input specifications: See documentation for details
Par = {}
Par["blocks"] = Spec.Blocks.copy() # Block loading structure
Par["nQ"] = (Spec.Frequency == "q").sum() # Number of quarterly series
Par["p"] = 1 # Number of lags in autoregressive of factor (same for all factors)
Par["r"] = np.ones((1, Spec.Blocks.shape[1])).astype(
np.int64) # Number of common factors for each block
# Par.r(1) =2;
# Display blocks
print("\n\n\n")
print("Table 3: Block Loading Structure")
print(
pd.DataFrame(data=Spec.Blocks,
index=Spec.SeriesName,
columns=Spec.BlockNames))
print("\n")
print("Estimating the dynamic factor model (DFM) \n\n")
T, N = X.shape
r = Par["r"].copy()
p = Par["p"]
nQ = Par["nQ"]
blocks = Par["blocks"].copy()
i_idio = np.append(np.ones(N - nQ), np.zeros(nQ)).reshape((-1, 1),
order="F") == 1
# R*Lambda = q; Contraints on the loadings of the quartrly variables
R_mat = np.array(
[2, -1, 0, 0, 0, 3, 0, -1, 0, 0, 2, 0, 0, -1, 0, 1, 0, 0, 0,
-1]).reshape((4, 5))
q = np.zeros((4, 1))
# Prepare data -----------------------------------------------------------
Mx = np.nanmean(X, axis=0)
Wx = np.nanstd(X, axis=0, ddof=1)
xNaN = (X - np.tile(Mx, (T, 1))) / np.tile(Wx, (T, 1))
# Initial Conditions------------------------------------------------------
optNaN = {}
optNaN["method"] = 2 # Remove leading and closing zeros
optNaN["k"] = 3 # Setting for filter(): See remNaN_spline
A, C, Q, R, Z_0, V_0 = InitCond(xNaN.copy(), r.copy(), p, blocks.copy(),
optNaN, R_mat.copy(), q, nQ, i_idio.copy())
# initialize EM loop values
previous_loglik = -np.inf
num_iter = 0
LL = [-np.inf]
converged = 0
# y for the estimation is with missing data
y = xNaN.copy().T
# EM LOOP ----------------------------------------------------------------
# The model can be written as
# y = C*Z + e;
# Z = A*Z(-1) + v
# where y is NxT, Z is (pr)xT, etc
# Remove the leading and ending nans
optNaN["method"] = 3
y_est, _ = remNaNs_spline(xNaN.copy(), optNaN)
y_est = y_est.T
max_iter = 5000
while num_iter < max_iter and not converged: # Loop until converges or max iter.
# Applying EM algorithm
C_new, R_new, A_new, Q_new, Z_0, V_0, loglik = EMstep(
y_est, A, C, Q, R, Z_0, V_0, r, p, R_mat, q, nQ, i_idio, blocks)
C = C_new.copy()
R = R_new.copy()
A = A_new.copy()
Q = Q_new.copy()
if num_iter > 2: # Check convergence
converged, decrease = em_converged(loglik, previous_loglik,
threshold, 1)
if (num_iter % 10) == 0 and num_iter > 0:
print("Now running the {}th iteration of max {}".format(
num_iter, max_iter))
print('Loglik: {} (% Change: {})'.format(
loglik, 100 * ((loglik - previous_loglik) / previous_loglik)))
LL.append(loglik)
previous_loglik = loglik
num_iter += 1
if num_iter < max_iter:
print('Successful: Convergence at {} interations'.format(num_iter))
else:
print('Stopped because maximum iterations reached')
# Final run of the Kalman filter
Zsmooth, _, _, _ = runKF(y, A, C, Q, R, Z_0, V_0)
Zsmooth = Zsmooth.T
x_sm = np.matmul(Zsmooth[1:, :], C.T) # Get smoothed X
# Loading the structure with the results --------------------------------
Res = {
"x_sm": x_sm.copy(),
"X_sm": np.tile(Wx, (T, 1)) * x_sm + np.tile(Mx, (T, 1)),
"Z": Zsmooth[1:, :].copy(),
"C": C.copy(),
"R": R.copy(),
"A": A.copy(),
"Q": Q.copy(),
"Mx": Mx.copy(),
"Wx": Wx.copy(),
"Z_0": Z_0.copy(),
"V_0": V_0.copy(),
"r": r,
"p": p,
"loglik": LL
}
# Display output
# Table with names and factor loadings
nQ = Par["nQ"]
nM = Spec.SeriesID.shape[0] - nQ
nLags = max(Par["p"], 5)
nFactors = np.sum(Par["r"])
print("\n Table 4: Factor Loadings for Monthly Series")
print(
pd.DataFrame(Res["C"][:nM, np.arange(0, nFactors * 5, 5)],
columns=Spec.BlockNames,
index=Spec.SeriesName[:nM]))
print("\n Table 5: Quarterly Loadings Sample (Global Factor)")
print(
pd.DataFrame(
Res["C"][(-1 - nQ + 1):, :5],
columns=['f1_lag0', 'f1_lag1', 'f1_lag2', 'f1_lag3', 'f1_lag4'],
index=Spec.SeriesName[-1 - nQ + 1:]))
# Table with AR model on factors (factors with AR parameter and variance of residuals)
A_terms = np.diag(Res["A"]).copy()
Q_terms = np.diag(Res["Q"]).copy()
print('\n Table 6: Autoregressive Coefficients on Factors')
print(
pd.DataFrame(
{
'AR_Coefficient': A_terms[np.arange(0, nFactors * 5,
5)].copy(),
'Variance_Residual': Q_terms[np.arange(0, nFactors * 5,
5)].copy()
},
index=Spec.BlockNames))
# Table with AR model idiosyncratic errors (factors with AR parameter and variance of residuals)
print('\n Table 7: Autoregressive Coefficients on Idiosyncratic Component')
A_len = A.shape[0]
Q_len = Q.shape[0]
A_index = np.hstack([
np.arange(nFactors * 5, nFactors * 5 + nM),
np.arange(nFactors * 5 + nM, A_len, 5)
])
Q_index = np.hstack([
np.arange(nFactors * 5, nFactors * 5 + nM),
np.arange(nFactors * 5 + nM, Q_len, 5)
])
print(
pd.DataFrame(
{
'AR_Coefficient': A_terms[A_index].copy(),
'Variance_Residual': Q_terms[Q_index].copy()
},
index=Spec.SeriesName))
return Res
def InitCond(x, r, p, blocks, optNaN, Rcon, q, nQ, i_idio):
#InitCond() Calculates initial conditions for parameter estimation
#
# Description:
# Given standardized data and model information, InitCond() creates
# initial parameter estimates. These are intial inputs in the EM
# algorithm, which re-estimates these parameters using Kalman filtering
# techniques.
#
#Inputs:
# - x: Standardized data
# - r: Number of common factors for each block
# - p: Number of lags in transition equation
# - blocks: Gives series loadings
# - optNaN: Option for missing values in spline. See remNaNs_spline() for details.
# - Rcon: Incorporates estimation for quarterly series (i.e. "tent structure")
# - q: Constraints on loadings for quarterly variables
# - NQ: Number of quarterly variables
# - i_idio: Logical. Gives index for monthly variables (1) and quarterly (0)
#
#Output:
# - A: Transition matrix
# - C: Observation matrix
# - Q: Covariance for transition equation residuals
# - R: Covariance for observation equation residuals
# - Z_0: Initial value of state
# - V_0: Initial value of covariance matrix
pC = Rcon.shape[1] # Gives 'tent' structure size (quarterly to monthly)
ppC = max(p, pC)
n_b = blocks.shape[1] # Number of blocks
xBal, indNaN = remNaNs_spline(x.copy(), optNaN) # Spline without NaNs
T, N = xBal.shape # Time T series number N
nM = N - nQ # Number of monthly series
xNaN = xBal.copy()
xNaN[indNaN] = np.nan # Set missing values equal to NaNs
res = xBal.copy(
) # Spline output equal to res Later this is used for residuals
resNaN = xNaN.copy() # Later used for residuals
# Initialize model coefficient output
C = None
A = None
Q = None
V_0 = None
# Set the first observations as NaNs: For quarterly-monthly aggreg. scheme
indNaN[:pC - 1, :] = np.True_
# Set the first observations as NaNs: For quarterly-monthly aggreg. scheme
for i in range(n_b): # Loop for each block
r_i = r[0, i].copy() # r_i = 1 when block is loaded
# Observation equation -----------------------------------------------
C_i = np.zeros(
(N, r_i * ppC)) # Initialize state variable matrix helper
idx_i = np.where(blocks[:,
i])[0] # Returns series index loading block i
idx_iM = idx_i[idx_i < nM] # Monthly series indicies for loaded blocks
idx_iQ = idx_i[idx_i >=
nM] # Quarterly series indicies for loaded blocks
# Returns eigenvector v w/largest eigenvalue d, CHECK: test if eig values are the same in Matlab
d, v = eig(np.cov(res[:, idx_iM], rowvar=False))
e_idx = np.where(d == np.max(d))[0]
d = d[e_idx]
v = v[:, e_idx]
# Flip sign for cleaner output. Gives equivalent results without this section
if np.sum(v) < 0:
v = -v
# For monthly series with loaded blocks (rows), replace with eigenvector
# This gives the loading
C_i[idx_iM, 0:r_i] = v.copy()
f = np.matmul(res[:, idx_iM],
v) # Data projection for eigenvector direction
F = np.array(f[(pC - 1):f.shape[0], :]).reshape((-1, 1))
# Lag matrix using loading. This is later used for quarterly series
for kk in range(1, max(p + 1, pC)):
F = np.concatenate((F, f[(pC - 1) - kk:f.shape[0] - kk, :]),
axis=1)
Rcon_i = np.kron(Rcon,
np.eye(r_i)) # Quarterly-monthly aggregation scheme
q_i = np.kron(q, np.zeros((r_i, 1)))
# Produces projected data with lag structure (so pC-1 fewer entries)
ff = F[:, 0:(r_i * pC)].copy()
for j in idx_iQ: # Loop for quarterly variables
# For series j, values are dropped to accommodate lag structure
xx_j = resNaN[(pC - 1):, j].copy()
if sum(~np.isnan(xx_j)) < (ff.shape[1] + 2):
xx_j = res[(pC - 1):, j].copy()
ff_j = ff[~np.isnan(xx_j), :].copy()
xx_j = xx_j[~np.isnan(xx_j)].reshape((-1, 1)).copy()
iff_j = np.linalg.inv(np.matmul(ff_j.T, ff_j))
Cc = np.matmul(np.matmul(iff_j, ff_j.T), xx_j)
a1 = np.matmul(iff_j, Rcon_i.T)
a2 = np.linalg.inv(np.matmul(np.matmul(Rcon_i, iff_j), Rcon_i.T))
a3 = np.matmul(Rcon_i, Cc) - q_i
# Spline data monthly to quarterly conversion
Cc = Cc - np.matmul(np.matmul(a1, a2), a3)
C_i[j, 0:pC * r_i] = Cc.T.copy() # Place in output matrix
# Zeros in first pC-1 entries (replace dropped from lag)
ff = np.concatenate([np.zeros((pC - 1, pC * r_i)), ff], axis=0)
# Residual Calculations
res = res - np.matmul(ff, C_i.T)
resNaN = res.copy()
resNaN[indNaN] = np.nan
# Combine past loadings together
if i == 0:
C = C_i.copy()
else:
C = np.hstack([C, C_i.copy()])
# Transition equation ------------------------------------------------
z = F[:, r_i - 1].copy() # Projected data (no lag)
Z = F[:, r_i:(r_i * (p + 1))].copy() # Data with lag 1
A_i = np.zeros(
(r_i * ppC, r_i * ppC)).T # Initialize transition matrix
A_temp = np.matmul(np.matmul(np.linalg.inv(np.matmul(Z.T, Z)), Z.T),
z) # OLS: gives coefficient value AR(p) process
A_i[:r_i, :r_i * p] = A_temp.T.copy()
A_i[r_i:, :r_i * (ppC - 1)] = np.eye(r_i * (ppC - 1))
Q_i = np.zeros((ppC * r_i, ppC * r_i))
e = z - np.matmul(Z, A_temp) # VAR residuals
Q_i[:r_i, :r_i] = np.cov(e, rowvar=False) # VAR covariance matrix
initV_i = np.reshape(
np.matmul(
np.linalg.inv(np.eye((r_i * ppC)**2) - np.kron(A_i, A_i)),
Q_i.flatten('F').reshape((-1, 1))), (r_i * ppC, r_i * ppC))
# Gives top left block for the transition matrix
if i == 0:
A = A_i.copy()
Q = Q_i.copy()
V_0 = initV_i.copy()
else:
A = block_diag(A, A_i)
Q = block_diag(Q, Q_i)
V_0 = block_diag(V_0, initV_i)
eyeN = np.eye(N)[:, i_idio.flatten('F')] # Used inside observation matrix
C = np.hstack([C, eyeN])
# Monthly-quarterly agreggation scheme
C = np.hstack([
C,
np.vstack([
np.zeros((nM, 5 * nQ)),
np.kron(np.eye(nQ),
np.array([1, 2, 3, 2, 1]).reshape((1, -1)))
])
])
# Initialize covariance matrix for transition matrix
R = np.diag(np.nanvar(resNaN, ddof=1, axis=0))
ii_idio = np.where(i_idio)[0] # Indicies for monthly variables
n_idio = ii_idio.shape[0] # Number of monthly variables
BM = np.zeros(
(n_idio, n_idio)) # Initialize monthly transition matrix values
SM = np.zeros(
(n_idio,
n_idio)) # Initialize monthly residual covariance matrix values
for i in range(n_idio): # Loop for monthly variables
# Set observation equation residual covariance matrix diagonal
R[ii_idio[i], ii_idio[i]] = 1e-4
# Subsetting series residuals for series i
res_i = resNaN[:, ii_idio[i]].copy()
# Returns number of leading/ending zeros
try:
leadZero = np.max(
np.where(np.arange(1, T +
1) == np.cumsum(np.isnan(res_i)))) + 1
except ValueError:
leadZero = None
try:
endZero = -(np.max(
np.where(
np.arange(1, T +
1) == np.cumsum(np.isnan(res_i[::-1])))[0]) + 1)
except ValueError:
endZero = None
# Truncate leading and ending zeros
res_i = res[:, ii_idio[i]].copy()
res_i = res_i[:endZero]
res_i = res_i[leadZero:].reshape((-1, 1), order="F")
# Linear regression: AR 1 process for monthly series residuals
BM[i, i] = np.matmul(
np.matmul(np.linalg.inv(np.matmul(res_i[:-1].T, res_i[:-1])),
res_i[:-1].T), res_i[1:])
SM[i, i] = np.cov(res_i[1:] - (res_i[:-1] * BM[i, i]), rowvar=False)
Rdiag = np.diag(R).copy()
sig_e = (Rdiag[nM:] / 19)
Rdiag[nM:] = 1e-4
R = np.diag(Rdiag).copy() # Covariance for obs matrix residuals
# For BQ, SQ
rho0 = np.array([[.1]])
temp = np.zeros((5, 5))
temp[0, 0] = 1
# Blocks for covariance matrices
SQ = np.kron(np.diag((1 - rho0[0, 0]**2) * sig_e), temp)
BQ = np.kron(
np.eye(nQ),
np.vstack([
np.hstack([rho0, np.zeros((1, 4))]),
np.hstack([np.eye(4), np.zeros((4, 1))])
]))
initViQ = np.matmul(np.linalg.inv(np.eye((5 * nQ)**2) - np.kron(BQ, BQ)),
SQ.reshape((-1, 1))).reshape((5 * nQ, 5 * nQ))
initViM = np.diag(1 / np.diag(np.eye(BM.shape[0]) - BM**2)) * SM
# Output
A = block_diag(A, BM, BQ)
Q = block_diag(Q, SM, SQ)
Z_0 = np.zeros((A.shape[0], 1))
V_0 = block_diag(V_0, initViM, initViQ)
return A, C, Q, R, Z_0, V_0