def __init__(self,
d_in,
d_out,
recognizer_hidden_size=7,
generative_hidden_size=4):
self.d_in = d_in
self.d_out = d_out
W1 = TensorNormal.generate(Ds=(d_in, 2, 2, 2,
recognizer_hidden_size))[0]
b1 = Normal.generate(D=recognizer_hidden_size)
W2 = TensorNormal.generate(Ds=(recognizer_hidden_size,
d_in + 2 + 2 + 2 + 9))[0]
b2 = Normal.generate(D=recognizer_hidden_size)
W3 = TensorNormal.generate(Ds=(recognizer_hidden_size,
recognizer_hidden_size, d_in))[0]
b3 = Normal.generate(D=d_in)
# Wr1 = TensorNormal.generate( Ds=( recognizer_hidden_size, d_out + 3 ) )[ 0 ]
# br1 = Normal.generate( D=recognizer_hidden_size )
# Wr2 = TensorNormal.generate( Ds=( d_in, recognizer_hidden_size ) )[ 0 ]
# br2 = Normal.generate( D=d_in )
Wg1 = TensorNormal.generate(Ds=(generative_hidden_size, d_in + 3))[0]
bg1 = Normal.generate(D=generative_hidden_size)
Wg2 = TensorNormal.generate(Ds=(d_out, generative_hidden_size))[0]
bg2 = Normal.generate(D=d_out)
# self.recognizer_params = [ W, b ]
self.recognizer_params = [(W1, b1), (W2, b2), (W3, b3)]
self.generative_hyper_params = [(Wg1, bg1), (Wg2, bg2)]
def sampleGenerativeParams(self, generative_hyper_params):
(Wg1, bg1), (Wg2, bg2) = generative_hyper_params
w1 = TensorNormal.sample(params=(Wg1, unitCovs(Wg1)), size=1)[0]
b1 = Normal.sample(params=(bg1, unitCov(bg1)), size=1)[0]
w2 = TensorNormal.sample(params=(Wg2, unitCovs(Wg2)), size=1)[0]
b2 = Normal.sample(params=(bg2, unitCov(bg2)), size=1)[0]
return (w1, b1), (w2, b2)
def log_likelihood( cls, x, params=None, nat_params=None ):
# Compute P( x | Ѳ; α )
assert ( params is None ) ^ ( nat_params is None )
A, sigma = params if params is not None else cls.natToStandard( *nat_params )
x, y = x
assert x.shape[ 0 ] == y.shape[ 0 ]
if( x.ndim != 1 ):
return sum( [ Normal.log_likelihood( _y, params=( A.dot( _x ), sigma ) ) for _x, _y in zip( x, y ) ] )
return Normal.log_likelihood( y, params=( A.dot( x ), sigma ) )
def KLPQ(self, q_params):
(Wg1, bg1), (Wg2, bg2) = q_params
ans = 0.0
ans += TensorNormal.KLDivergence(**covsHelp(Wg1))
ans += Normal.KLDivergence(**covHelp(bg1))
ans += TensorNormal.KLDivergence(**covsHelp(Wg2))
ans += Normal.KLDivergence(**covHelp(bg2))
return ans
def sufficientStats(cls, x, constParams=None):
# Compute T( x )
if (cls.dataN(x) > 1):
t = (0, 0, 0, 0, 0)
for mu, sigma in zip(*x):
t = np.add(t, cls.sufficientStats((mu, sigma)))
return t
t1, t2 = Normal.standardToNat(*x)
t3, t4, t5 = Normal.log_partition(params=x, split=True)
return t1, t2, -t3, -t4, -t5
def sample(cls, params=None, nat_params=None, size=1):
# Sample from P( x | Ѳ; α )
assert (params is None) ^ (nat_params is None)
mu_0, kappa, psi, nu, _ = params if params is not None else cls.natToStandard(
*nat_params)
if (size > 1):
ans = tuple(
list(
zip(*[
cls.unpackSingleSample(
cls.sample(
params=params, nat_params=nat_params, size=1))
for _ in range(size)
])))
mu, sigma = ans
ans = (np.array(mu), np.array(sigma))
else:
sigma = InverseWishart.sample(params=(psi, nu))
mu = Normal.sample(
params=(mu_0,
InverseWishart.unpackSingleSample(sigma) / kappa))
ans = (mu, sigma)
cls.checkShape(ans)
return ans
def log_likelihoodRavel(cls, x, params=None, nat_params=None):
assert (params is None) ^ (nat_params is None)
M, covs = params if params is not None else cls.natToStandard(
*nat_params)
assert x.shape[1:] == M.shape
fullMu = M.ravel()
ans = 0.0
for _x in x:
fullCov = reduce(lambda x, y: np.kron(x, y), covs)
ans += Normal.log_likelihood(_x.ravel(), params=(fullMu, fullCov))
return ans
def log_likelihood(cls, x, params=None, nat_params=None):
# Compute P( x | Ѳ; α )
assert (params is None) ^ (nat_params is None)
mu_0, kappa, psi, nu, _ = params if params is not None else cls.natToStandard(
*nat_params)
if (cls.dataN(x) > 1):
return sum([
cls.log_likelihood((mu, sigma),
params=params,
nat_params=nat_params)
for mu, sigma in zip(*x)
])
mu, sigma = x
return InverseWishart.log_likelihood( sigma, params=( psi, nu ) ) + \
Normal.log_likelihood( mu, params=( mu_0, sigma / kappa ) )
def sample( cls, x=None, params=None, nat_params=None, size=1 ):
# Sample from P( x | Ѳ; α )
assert ( params is None ) ^ ( nat_params is None )
A, sigma = params if params is not None else cls.natToStandard( *nat_params )
D = A.shape[ 1 ]
if( x is None ):
x = np.array( [ Normal.unpackSingleSample( Normal.sample( params=( np.zeros( D ), np.eye( D ) ), size=1 ) ) for _ in range( size ) ] )
y = np.array( [ Normal.unpackSingleSample( Normal.sample( params=( A.dot( _x ), sigma ), size=1 ) ) for _x in x ] )
ans = ( x, y )
cls.checkShape( ans )
return ans
ans = Normal.sample( params=( A.dot( x ), sigma ), size=size )
Normal.checkShape( ans )
return ans
def log_likelihood( cls, x, params=None, nat_params=None ):
# Compute P( x | Ѳ; α )
assert ( params is None ) ^ ( nat_params is None )
A, sigma = params if params is not None else cls.natToStandard( *nat_params )
xs, ys = x
assert ( isinstance( xs, tuple ) or isinstance( xs, list ) ) and isinstance( ys, np.ndarray )
assert len( xs ) == len( A.shape ) - 1 and ys.ndim == 2
for x in xs:
assert isinstance( x, np.ndarray ) and x.shape == ys.shape
N = len( A.shape )
ind = string.ascii_letters[ :N ]
tInd = string.ascii_letters[ N ]
contract = ind + ',' + tInd + ( ',' + tInd ).join( [ l for l in ind[ 1: ] ] ) + '->' + tInd + ind[ 0 ]
mus = np.einsum( contract, A, *xs, optimize=( N > 2 ) )
return sum( [ Normal.log_likelihood( y, params=( mu, sigma ) ) for mu, y in zip( mus, ys ) ] )
def sample(cls, params=None, nat_params=None, size=1):
# Sample from P( x | Ѳ; α )
assert (params is None) ^ (nat_params is None)
M, covs = params if params is not None else cls.natToStandard(
*nat_params)
cov_chols = [np.linalg.cholesky(cov) for cov in covs]
shapes = [cov.shape[0] for cov in covs]
N = len(shapes)
total_dim = np.prod([size] + shapes)
X = Normal.generate(D=1, size=total_dim).reshape([size] + shapes)
ind1 = string.ascii_letters[:N]
ind2 = string.ascii_letters[N:N * 2]
t = string.ascii_letters[N * 2]
contract = t + ind1 + ',' + ','.join(
[a + b for a, b in zip(ind1, ind2)]) + '->' + t + ind2
ans = M + np.einsum(contract, X, *cov_chols)
cls.checkShape(ans)
return ans
def sample( cls, xs=None, params=None, nat_params=None, size=1 ):
# Sample from P( x | Ѳ; α )
assert ( params is None ) ^ ( nat_params is None )
A, sigma = params if params is not None else cls.natToStandard( *nat_params )
D = sigma.shape[ 0 ]
N = len( A.shape )
if( xs is None ):
xs = [ np.random.random( ( size, D ) ) for _ in range( N - 1 ) ]
returnBoth = True
else:
returnBoth = False
ind = string.ascii_letters[ :N ]
tInd = string.ascii_letters[ N ]
contract = ind + ',' + tInd + ( ',' + tInd ).join( [ l for l in ind[ 1: ] ] ) + '->' + tInd + ind[ 0 ]
mus = np.einsum( contract, A, *xs, optimize=( N > 2 ) )
ys = np.array( [ Normal.sample( params=( mu, sigma ) ) for mu in mus ] )
return xs, ys if returnBoth else ys