def _set_expected_stats_latents(self, smoothed_mus, smoothed_sigmas):
sub_pops = self.obs_scheme.sub_pops
obs_pops = self.obs_scheme.obs_pops
obs_time = self.obs_scheme.obs_time
idx_grp = self.obs_scheme.idx_grp
obs_idx = self.obs_scheme.obs_idx
if obs_time.size > 1:
T = np.zeros(len(idx_grp))
Ex = np.zeros(self.n)
ExxT = np.zeros((self.n, self.n))
Exe = np.zeros((self.n, len(idx_grp)))
ExxTj = np.zeros((self.n, self.n, len(idx_grp)))
ExxTe = np.zeros((self.n + 1, self.n + 1, len(idx_grp)))
for i in range(obs_time.size):
ts = range(obs_time[0]) if i == 0 else range(
obs_time[i - 1], obs_time[i])
x = smoothed_mus[ts, :]
sx = np.sum(x, 0)
sxxT = smoothed_sigmas[ts, :, :].sum(0) + x.T.dot(x)
for j in obs_idx[i]:
Exe[:, j] += sx
ExxTj[:, :, j] += sxxT
T[j] += len(ts)
Ex += sx
ExxT += sxxT
for j in range(len(idx_grp)):
ExxTe[:, :, j] = blockarray(
[[ExxTj[:, :, j],
np.atleast_2d(Exe[:, j]).T],
[np.atleast_2d(Exe[:, j]),
np.atleast_2d(T[j])]])
else:
Ex = smoothed_mus.sum(0)
ExxT = smoothed_sigmas.sum(0) + smoothed_mus.T.dot(smoothed_mus)
E_xt_xtT = \
ExxT - (smoothed_sigmas[-1]
+ np.outer(smoothed_mus[-1],smoothed_mus[-1]))
E_xtp1_xtp1T = \
ExxT - (smoothed_sigmas[0]
+ np.outer(smoothed_mus[0], smoothed_mus[0]))
if self.model.emission_distn.affine:
ExxT = blockarray([[ExxT, np.atleast_2d(Ex).T],
[np.atleast_2d(Ex),
np.atleast_2d(self.T)]])
if not obs_time.size > 1:
ExxTe = ExxT
return ExxT, E_xt_xtT, E_xtp1_xtp1T, ExxTe
def _set_expected_stats_latents(self, smoothed_mus, smoothed_sigmas):
sub_pops = self.obs_scheme.sub_pops
obs_pops = self.obs_scheme.obs_pops
obs_time = self.obs_scheme.obs_time
idx_grp = self.obs_scheme.idx_grp
obs_idx = self.obs_scheme.obs_idx
if obs_time.size > 1:
T = np.zeros(len(idx_grp))
Ex = np.zeros(self.n)
ExxT = np.zeros((self.n,self.n))
Exe = np.zeros((self.n, len(idx_grp)))
ExxTj = np.zeros((self.n, self.n, len(idx_grp)))
ExxTe = np.zeros((self.n+1, self.n+1, len(idx_grp)))
for i in range(obs_time.size):
ts = range(obs_time[0]) if i==0 else range(obs_time[i-1], obs_time[i])
x = smoothed_mus[ts,:]
sx = np.sum(x,0)
sxxT = smoothed_sigmas[ts,:,:].sum(0) + x.T.dot(x)
for j in obs_idx[i]:
Exe[:,j] += sx
ExxTj[:,:,j] += sxxT
T[j] += len(ts)
Ex += sx
ExxT += sxxT
for j in range(len(idx_grp)):
ExxTe[:,:,j] = blockarray([[ExxTj[:,:,j],np.atleast_2d(Exe[:,j]).T],
[np.atleast_2d(Exe[:,j]),np.atleast_2d(T[j])]])
else:
Ex = smoothed_mus.sum(0)
ExxT = smoothed_sigmas.sum(0) + smoothed_mus.T.dot(smoothed_mus)
E_xt_xtT = \
ExxT - (smoothed_sigmas[-1]
+ np.outer(smoothed_mus[-1],smoothed_mus[-1]))
E_xtp1_xtp1T = \
ExxT - (smoothed_sigmas[0]
+ np.outer(smoothed_mus[0], smoothed_mus[0]))
if self.model.emission_distn.affine:
ExxT = blockarray([[ExxT,np.atleast_2d(Ex).T],
[np.atleast_2d(Ex),np.atleast_2d(self.T)]])
if not obs_time.size > 1:
ExxTe = ExxT
return ExxT, E_xt_xtT, E_xtp1_xtp1T, ExxTe
def log_likelihood(self, xy):
assert isinstance(xy, (tuple, np.ndarray))
A, sigma, D = self.A, self.sigma, self.D_out
x, y = (xy[:, :-D], xy[:, -D:]) if isinstance(xy, np.ndarray) else xy
if self.affine:
A, b = A[:, :-1], A[:, -1]
sigma_inv, L = inv_psd(sigma, return_chol=True)
parammat = -1. / 2 * blockarray([[
A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)
], [-sigma_inv.dot(A), sigma_inv]])
contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->'
if isinstance(xy, np.ndarray):
out = np.einsum(contract, xy.dot(parammat), xy)
else:
out = np.einsum(contract, x.dot(parammat[:-D, :-D]), x)
out += np.einsum(contract, y.dot(parammat[-D:, -D:]), y)
out += 2 * np.einsum(contract, x.dot(parammat[:-D, -D:]), y)
out -= D / 2 * np.log(2 * np.pi) + np.log(np.diag(L)).sum()
if self.affine:
out += y.dot(sigma_inv).dot(b)
out -= x.dot(A.T).dot(sigma_inv).dot(b)
out -= 1. / 2 * b.dot(sigma_inv).dot(b)
return out
def log_likelihood(self,xy):
assert isinstance(xy, (tuple,np.ndarray))
A, sigma, D = self.A, self.sigma, self.D_out
x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy,np.ndarray) else xy
if self.affine:
A, b = A[:,:-1], A[:,-1]
sigma_inv, L = inv_psd(sigma, return_chol=True)
parammat = -1./2 * blockarray([
[A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)],
[-sigma_inv.dot(A), sigma_inv]])
contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->'
if isinstance(xy, np.ndarray):
out = np.einsum(contract,xy.dot(parammat),xy)
else:
out = np.einsum(contract,x.dot(parammat[:-D,:-D]),x)
out += np.einsum(contract,y.dot(parammat[-D:,-D:]),y)
out += 2*np.einsum(contract,x.dot(parammat[:-D,-D:]),y)
out -= D/2*np.log(2*np.pi) + np.log(np.diag(L)).sum()
if self.affine:
out += y.dot(sigma_inv).dot(b)
out -= x.dot(A.T).dot(sigma_inv).dot(b)
out -= 1./2*b.dot(sigma_inv).dot(b)
return out
def _param_matrix(o):
D, A, sigma = o.D, o.A, o.sigma
sigma_inv = np.linalg.inv(sigma)
parammat = -1./2 * blockarray([
[A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)],
[-sigma_inv.dot(A), sigma_inv]
])
normalizer = D/2*np.log(2*np.pi) + np.log(np.diag(np.linalg.cholesky(sigma))).sum()
return parammat, normalizer
def _param_matrix(o):
D, A, sigma = o.D, o.A, o.sigma
sigma_inv = np.linalg.inv(sigma)
parammat = -1. / 2 * blockarray([[
A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)
], [-sigma_inv.dot(A), sigma_inv]])
normalizer = D / 2 * np.log(2 * np.pi) + np.log(
np.diag(np.linalg.cholesky(sigma))).sum()
return parammat, normalizer
def expected_log_likelihood(self, xy=None, stats=None):
# TODO test values, test for the affine case
assert isinstance(xy, (tuple, np.ndarray)) ^ isinstance(stats, tuple)
D = self.D_out
E_Sigmainv, E_Sigmainv_A, E_AT_Sigmainv_A, E_logdetSigmainv = \
mniw_expectedstats(
*self._natural_to_standard(self.mf_natural_hypparam))
if self.affine:
E_Sigmainv_A, E_Sigmainv_b = \
E_Sigmainv_A[:,:-1], E_Sigmainv_A[:,-1]
E_AT_Sigmainv_A, E_AT_Sigmainv_b, E_bT_Sigmainv_b = \
E_AT_Sigmainv_A[:-1,:-1], E_AT_Sigmainv_A[:-1,-1], \
E_AT_Sigmainv_A[-1,-1]
if xy is not None:
x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy, np.ndarray) \
else xy
parammat = -1./2 * blockarray([
[E_AT_Sigmainv_A, -E_Sigmainv_A.T],
[-E_Sigmainv_A, E_Sigmainv]])
contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->'
if isinstance(xy, np.ndarray):
out = np.einsum('ni,ni->n', xy.dot(parammat), xy)
else:
out = np.einsum(contract,x.dot(parammat[:-D,:-D]),x)
out += np.einsum(contract,y.dot(parammat[-D:,-D:]),y)
out += 2*np.einsum(contract,x.dot(parammat[:-D,-D:]),y)
out += -D/2*np.log(2*np.pi) + 1./2*E_logdetSigmainv
if self.affine:
out += y.dot(E_Sigmainv_b)
out -= x.dot(E_AT_Sigmainv_b)
out -= 1./2 * E_bT_Sigmainv_b
else:
if self.affine:
Ey, Ex = stats[:2]
yyT, yxT, xxT, n = stats[-4:]
contract = 'ij,nij->n' if yyT.ndim == 3 else 'ij,ij->'
out = -1./2 * np.einsum(contract, E_AT_Sigmainv_A, xxT)
out += np.einsum(contract, E_Sigmainv_A, yxT)
out += -1./2 * np.einsum(contract, E_Sigmainv, yyT)
out += -D/2*np.log(2*np.pi) + n/2.*E_logdetSigmainv
if self.affine:
out += Ey.dot(E_Sigmainv_b)
out -= Ex.dot(E_AT_Sigmainv_b)
out -= 1./2 * E_bT_Sigmainv_b
return out
def expected_log_likelihood(self, xy=None, stats=None):
# TODO test values, test for the affine case
assert isinstance(xy, (tuple, np.ndarray)) ^ isinstance(stats, tuple)
D = self.D_out
E_Sigmainv, E_Sigmainv_A, E_AT_Sigmainv_A, E_logdetSigmainv = \
mniw_expectedstats(
*self._natural_to_standard(self.mf_natural_hypparam))
if self.affine:
E_Sigmainv_A, E_Sigmainv_b = \
E_Sigmainv_A[:,:-1], E_Sigmainv_A[:,-1]
E_AT_Sigmainv_A, E_AT_Sigmainv_b, E_bT_Sigmainv_b = \
E_AT_Sigmainv_A[:-1,:-1], E_AT_Sigmainv_A[:-1,-1], \
E_AT_Sigmainv_A[-1,-1]
if xy is not None:
x, y = (xy[:,:-D], xy[:,-D:]) if isinstance(xy, np.ndarray) \
else xy
parammat = -1. / 2 * blockarray([[
E_AT_Sigmainv_A, -E_Sigmainv_A.T
], [-E_Sigmainv_A, E_Sigmainv]])
contract = 'ni,ni->n' if x.ndim == 2 else 'i,i->'
if isinstance(xy, np.ndarray):
out = np.einsum('ni,ni->n', xy.dot(parammat), xy)
else:
out = np.einsum(contract, x.dot(parammat[:-D, :-D]), x)
out += np.einsum(contract, y.dot(parammat[-D:, -D:]), y)
out += 2 * np.einsum(contract, x.dot(parammat[:-D, -D:]), y)
out += -D / 2 * np.log(2 * np.pi) + 1. / 2 * E_logdetSigmainv
if self.affine:
out += y.dot(E_Sigmainv_b)
out -= x.dot(E_AT_Sigmainv_b)
out -= 1. / 2 * E_bT_Sigmainv_b
else:
if self.affine:
Ey, Ex = stats[:2]
yyT, yxT, xxT, n = stats[-4:]
contract = 'ij,nij->n' if yyT.ndim == 3 else 'ij,ij->'
out = -1. / 2 * np.einsum(contract, E_AT_Sigmainv_A, xxT)
out += np.einsum(contract, E_Sigmainv_A, yxT)
out += -1. / 2 * np.einsum(contract, E_Sigmainv, yyT)
out += -D / 2 * np.log(2 * np.pi) + n / 2. * E_logdetSigmainv
if self.affine:
out += Ey.dot(E_Sigmainv_b)
out -= Ex.dot(E_AT_Sigmainv_b)
out -= 1. / 2 * E_bT_Sigmainv_b
return out
def _get_scaled_statistics(self, data, precisions):
assert isinstance(data, (list, tuple, np.ndarray))
if isinstance(data, list):
return sum((self._get_scaled_statistics(d, p)
for d, p in zip(data, precisions)),
self._empty_statistics())
elif isinstance(data, tuple):
x, y = data
bad = np.isnan(x).any(1) | np.isnan(y).any(1)
x, y = x[~bad], y[~bad]
precisions = precisions[~bad]
sqrt_prec = np.sqrt(precisions)
n, D = y.shape
if self.affine:
x = np.column_stack((x, np.ones(n)))
# Scale by the precision
# xs = x * sqrt_prec[:, na]
# ys = y * sqrt_prec[:, na]
xs = x * np.tile(sqrt_prec[:, None], (1, x.shape[1]))
ys = y * np.tile(sqrt_prec[:, None], (1, D))
xxT, yxT, yyT = xs.T.dot(xs), ys.T.dot(xs), ys.T.dot(ys)
return np.array([yyT, yxT, xxT, n])
else:
# data passed in like np.hstack((x, y))
# x, y = data[:,:-self.D_out], data[:,-self.D_out:]
# return self._get_scaled_statistics((x, y), precisions)
bad = np.isnan(data).any(1)
data = data[~bad]
precisions = precisions[~bad]
n, D = data.shape[0], self.D_out
# This tile call is suboptimal but without it we can hit issues
# with strided data, as in autoregressive models.
scaled_data = data * np.tile(precisions[:, None],
(1, data.shape[1]))
statmat = scaled_data.T.dot(data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = scaled_data.sum(0)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT, x[:, na]],
[x[na, :],
np.atleast_2d(precisions.sum())]])
yxT = np.hstack((yxT, y[:, na]))
return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self, data, weights):
assert isinstance(data, (list, tuple, np.ndarray))
if isinstance(data, list):
return sum((self._get_statistics(d) for d in data),
self._empty_statistics())
elif isinstance(data, tuple):
x, y = data
bad = np.isnan(x).any(1) | np.isnan(y).any(1)
x, y, weights = x[~bad], y[~bad], weights[~bad]
n, D = weights.sum(), y.shape[1]
wx = weights[:, na] * x
xxT, yxT, yyT = \
x.T.dot(wx), y.T.dot(wx), y.T.dot(weights[:,na]*y)
if self.affine:
x, y = weights.dot(x), weights.dot(y)
xxT = blockarray([[xxT, x[:, na]],
[x[na, :], np.atleast_2d(n)]])
yxT = np.hstack((yxT, y[:, na]))
return np.array([yyT, yxT, xxT, n])
else:
# data passed in like np.hstack((x, y))
gi = ~np.isnan(data).any(1)
data, weights = data[gi], weights[gi]
n, D = weights.sum(), self.D_out
statmat = data.T.dot(weights[:, na] * data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = weights.dot(data)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT, x[:, na]],
[x[na, :], np.atleast_2d(n)]])
yxT = np.hstack((yxT, y[:, na]))
return np.array([yyT, yxT, xxT, n])
def _stats_ensure_array(stats):
if isinstance(stats, np.ndarray):
return stats
affine = len(stats) > 4
yyT, yxT, xxT, n = stats[-4:]
if affine:
y, x = stats[:2]
yxT = np.hstack((yxT, y[:, None]))
xxT = blockarray([[xxT, x[:, None]], [x[None, :], 1.]])
return np.array([yyT, yxT, xxT, n])
def _stats_ensure_array(stats):
if isinstance(stats, np.ndarray):
return stats
affine = len(stats) > 4
yyT, yxT, xxT, n = stats[-4:]
if affine:
y, x = stats[:2]
yxT = np.hstack((yxT, y[:,None]))
xxT = blockarray([[xxT, x[:,None]], [x[None,:], 1.]])
return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self,data,weights):
assert isinstance(data, (list, tuple, np.ndarray))
if isinstance(data,list):
return sum((self._get_statistics(d) for d in data),
self._empty_statistics())
elif isinstance(data, tuple):
x, y = data
bad = np.isnan(x).any(1) | np.isnan(y).any(1)
x, y, weights = x[~bad], y[~bad], weights[~bad]
n, D = weights.sum(), y.shape[1]
wx = weights[:,na]*x
xxT, yxT, yyT = \
x.T.dot(wx), y.T.dot(wx), y.T.dot(weights[:,na]*y)
if self.affine:
x, y = weights.dot(x), weights.dot(y)
xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]])
yxT = np.hstack((yxT,y[:,na]))
return np.array([yyT, yxT, xxT, n])
else:
# data passed in like np.hstack((x, y))
gi = ~np.isnan(data).any(1)
data, weights = data[gi], weights[gi]
n, D = weights.sum(), self.D_out
statmat = data.T.dot(weights[:,na]*data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = weights.dot(data)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]])
yxT = np.hstack((yxT,y[:,na]))
return np.array([yyT, yxT, xxT, n])
def _get_scaled_statistics(self, data, precisions):
assert isinstance(data, (list, tuple, np.ndarray))
if isinstance(data,list):
return sum((self._get_scaled_statistics(d, p) for d, p in zip(data, precisions)),
self._empty_statistics())
elif isinstance(data, tuple):
x, y = data
bad = np.isnan(x).any(1) | np.isnan(y).any(1)
x, y = x[~bad], y[~bad]
precisions = precisions[~bad]
sqrt_prec = np.sqrt(precisions)
n, D = y.shape
if self.affine:
x = np.column_stack((x, np.ones(n)))
# Scale by the precision
# xs = x * sqrt_prec[:, na]
# ys = y * sqrt_prec[:, na]
xs = x * np.tile(sqrt_prec[:, None], (1, x.shape[1]))
ys = y * np.tile(sqrt_prec[:, None], (1, D))
xxT, yxT, yyT = xs.T.dot(xs), ys.T.dot(xs), ys.T.dot(ys)
return np.array([yyT, yxT, xxT, n])
else:
# data passed in like np.hstack((x, y))
# x, y = data[:,:-self.D_out], data[:,-self.D_out:]
# return self._get_scaled_statistics((x, y), precisions)
bad = np.isnan(data).any(1)
data = data[~bad]
precisions = precisions[~bad]
n, D = data.shape[0], self.D_out
# This tile call is suboptimal but without it we can hit issues
# with strided data, as in autoregressive models.
scaled_data = data * np.tile(precisions[:,None], (1, data.shape[1]))
statmat = scaled_data.T.dot(data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = scaled_data.sum(0)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT, x[:,na]],
[x[na,:], np.atleast_2d(precisions.sum())]])
yxT = np.hstack((yxT, y[:,na]))
return np.array([yyT, yxT, xxT, n])
def _get_statistics(self,data):
if isinstance(data,list):
return sum((self._get_statistics(d) for d in data),
self._empty_statistics())
else:
data = data[~np.isnan(data).any(1)]
n, D = data.shape[0], self.D_out
statmat = data.T.dot(data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = data.sum(0)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]])
yxT = np.hstack((yxT,y[:,na]))
return np.array([yyT, yxT, xxT, n])
def _get_weighted_statistics(self,data,weights):
if isinstance(data,list):
return sum((self._get_statistics(d) for d in data),
self._empty_statistics())
else:
gi = ~np.isnan(data).any(1)
data, weights = data[gi], weights[gi]
n, D = weights.sum(), self.D_out
statmat = data.T.dot(weights[:,na]*data)
xxT, yxT, yyT = \
statmat[:-D,:-D], statmat[-D:,:-D], statmat[-D:,-D:]
if self.affine:
xy = weights.dot(data)
x, y = xy[:-D], xy[-D:]
xxT = blockarray([[xxT,x[:,na]],[x[na,:],np.atleast_2d(n)]])
yxT = np.hstack((yxT,y[:,na]))
return np.array([yyT, yxT, xxT, n])
def log_likelihood(self,xy):
A, sigma, D = self.A, self.sigma, self.D_out
x, y = xy[:,:-D], xy[:,-D:]
if self.affine:
A, b = A[:,:-1], A[:,-1]
sigma_inv, L = inv_psd(sigma, return_chol=True)
parammat = -1./2 * blockarray([
[A.T.dot(sigma_inv).dot(A), -A.T.dot(sigma_inv)],
[-sigma_inv.dot(A), sigma_inv]
])
out = np.einsum('ni,ni->n',xy.dot(parammat),xy)
out -= D/2*np.log(2*np.pi) + np.log(np.diag(L)).sum()
if self.affine:
out += y.dot(sigma_inv).dot(b)
out -= x.dot(A.T).dot(sigma_inv).dot(b)
out -= 1./2*b.dot(sigma_inv).dot(b)
return out
