def update_global_params_VB(self, SS, **kwargs):
''' Update global parameter theta to optimize VB objective.
Post condition
--------------
Attributes rho,omega,theta set to optimal value given suff stats.
'''
nGlobalIters = 2
nNode = SS.NodeStateCount.shape[0]
if not hasattr(self, 'rho') or self.rho.size != SS.K:
self.rho = OptimizerRhoOmegaBetter.make_initrho(
SS.K, nNode, self.gamma)
self.omega = OptimizerRhoOmegaBetter.make_initomega(
SS.K, nNode, self.gamma)
# Update theta first, so it reflects most recent NodeStateCounts
self.theta, self.thetaRem = updateThetaAndThetaRem(
SS, rho=self.rho, alpha=self.alpha, gamma=self.gamma)
# Now, alternatively update rho and theta...
for giter in range(nGlobalIters):
self.rho, self.omega = updateRhoOmega(
theta=self.theta, thetaRem=self.thetaRem,
initrho=self.rho, omega=self.omega,
alpha=self.alpha, gamma=self.gamma)
self.theta, self.thetaRem = updateThetaAndThetaRem(
SS, rho=self.rho, alpha=self.alpha, gamma=self.gamma)
def _find_optimum_rhoomega(self, SS, **kwargs):
''' Run numerical optimization to find optimal rho, omega parameters
Args
--------
SS : bnpy SuffStatBag, with K components
Returns
--------
rho : 1D array, length K
omega : 1D array, length K
'''
if hasattr(self, 'rho') and self.rho.size == SS.K:
if 'sortorder' in kwargs:
# Adjust initialization to account for new ordering.
beta = OptimizerRhoOmegaBetter.rho2beta_active(self.rho)
beta = beta[kwargs['sortorder']]
initrho = OptimizerRhoOmegaBetter.beta2rho(beta, SS.K)
else:
initrho = self.rho
else:
initrho = None # default initialization
# Always use from-scratch init for omega
initomega = OptimizerRhoOmegaBetter.make_initomega(
SS.K, SS.nDoc, self.gamma)
try:
if hasattr(SS, 'sumLogPiRemVec'):
sumLogPiRemVec = SS.sumLogPiRemVec
else:
sumLogPiRemVec = np.zeros(SS.K)
sumLogPiRemVec[-1] = SS.sumLogPiRem
rho, omega, f, Info = OptimizerRhoOmegaBetter.\
find_optimum_multiple_tries(
sumLogPiActiveVec=SS.sumLogPi,
sumLogPiRemVec=sumLogPiRemVec,
nDoc=SS.nDoc,
gamma=self.gamma,
alpha=self.alpha,
initrho=initrho,
initomega=initomega,
do_grad_omega=0,
Log=Log)
except ValueError as error:
if str(error).count('FAILURE') == 0:
raise error
if hasattr(self, 'rho') and self.rho.size == SS.K:
Log.error('***** Optim failed. Remain at cur val. ' +
str(error))
rho = self.rho
omega = initomega
else:
Log.error('***** Optim failed. Set to default init. ' +
str(error))
rho = OptimizerRhoOmegaBetter.make_initrho(
SS.K, SS.nDoc, self.gamma)
omega = initomega
return rho, omega
def _beta2rhoomega(beta, K, nDoc=10, gamma=10):
''' Find vectors rho, omega that are probable given beta
Returns
--------
rho : 1D array, size K
omega : 1D array, size K
'''
assert beta.size == K or beta.size == K + 1
rho = OptimizerRhoOmegaBetter.beta2rho(beta, K)
omega = OptimizerRhoOmegaBetter.make_initomega(K, nDoc, gamma)
return rho, omega
def updateThetaAndThetaRem(
SS=None, K=None, NodeStateCount=None, rho=None,
alpha=1.0, gamma=10.0):
''' Update parameters theta to maximize objective given suff stats.
Returns
---------
theta : 2D array, nNodes x K
thetaRem : scalar
'''
if K is None:
K = SS.K
if NodeStateCount is None:
NodeStateCount = SS.NodeStateCount
nNodes = NodeStateCount.shape[0]
if rho is None or rho.size != K:
rho = OptimizerRhoOmegaBetter.make_initrho(K, nNodes, gamma)
# Calculate E_q[alpha * Beta_l] for l = 1, ..., K+1
Ebeta = StickBreakUtil.rho2beta(rho, returnSize='K')
alphaEbeta = alpha * Ebeta
alphaEbetaRem = alpha * (1- Ebeta.sum())
theta = alphaEbeta + NodeStateCount
thetaRem = alphaEbetaRem
return theta, thetaRem
def f_DocTopicCount(rho=None,
omega=None,
betaK=None,
DocTopicCount=None,
alpha=None,
gamma=None,
**kwargs):
''' Evaluate the objective f for rho/omega optimization.
Returns
-------
f : scalar
'''
K = rho.size
sumLogPiActiveVec, sumLogPiRemVec, LP = DocTopicCount_to_sumLogPi(
rho=rho,
omega=omega,
DocTopicCount=DocTopicCount,
alpha=alpha,
gamma=gamma,
**kwargs)
f = OptimizerRhoOmegaBetter.negL_rhoomega(
rho=rho,
omega=omega,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
alpha=alpha,
gamma=gamma,
nDoc=DocTopicCount.shape[0],
approx_grad=1)
return f
def updateRhoOmega(theta=None,
thetaRem=None,
initrho=None,
omega=None,
alpha=0.5,
gamma=10,
logFunc=None):
''' Update rho, omega via numerical optimization.
Will set vector omega to reasonable fixed value,
and do gradient descent to optimize the vector rho.
Returns
-------
rho : 1D array, size K
omega : 1D array, size K
'''
nDoc = theta.shape[0]
K = theta.shape[1]
# Verify initial rho
assert initrho is not None
assert initrho.size == K
# Verify initial omega
assert omega is not None
assert omega.size == K
# Compute summaries of theta needed to update rho
# sumLogPi : 1D array, size K
# sumLogPiRem : scalar
digammasumtheta = digamma(theta.sum(axis=1) + thetaRem)
ElogPi = digamma(theta) - digammasumtheta[:, np.newaxis]
sumLogPi = np.sum(ElogPi, axis=0)
ElogPiRem = digamma(thetaRem) - digammasumtheta
sumLogPiRem = np.sum(ElogPiRem)
# Do the optimization
try:
rho, omega, fofu, Info = \
OptimizerRhoOmegaBetter.find_optimum_multiple_tries(
nDoc=nDoc,
sumLogPiActiveVec=sumLogPi,
sumLogPiRem=sumLogPiRem,
gamma=gamma,
alpha=alpha,
initrho=initrho,
initomega=omega,
do_grad_omega=0,
do_grad_rho=1)
except ValueError as error:
if logFunc:
logFunc('***** Rho optim failed. Remain at cur val. ' + \
str(error))
rho = initrho
assert rho.size == K
assert omega.size == K
return rho, omega
def L_hdp(beta, omega, Tvec, alpha):
''' Compute top-tier of hdp bound.
'''
K = omega.size
assert K == beta.size
assert K == Tvec.size
rho = OROB.beta2rho(beta, K)
eta1 = rho * omega
eta0 = (1 - rho) * omega
digamma_omega = digamma(omega)
digamma_eta1 = digamma(eta1)
digamma_eta0 = digamma(eta0)
ElogU = digamma_eta1 - digamma_omega
Elog1mU = digamma_eta0 - digamma_omega
Lscore = \
np.sum(gammaln(eta1) + gammaln(eta0) - gammaln(omega)) \
+ np.inner(nDoc + 1 - eta1, ElogU) \
+ np.inner(nDoc * OROB.kvec(K) + gamma - eta0, Elog1mU) \
+ alpha * np.inner(beta, Tvec)
return Lscore
def evalELBOandPrint(nDoc=None,
theta=None,
thetaRem=None,
DocTopicCount=None,
sumLogPiActiveVec=None,
sumLogPiRemVec=None,
alpha=None,
gamma=None,
rho=None,
omega=None,
msg='',
f=None,
**kwargs):
''' Check on the objective.
'''
L = calcELBO_IgnoreTermsConstWRTrhoomegatheta(
nDoc=nDoc,
alpha=alpha,
gamma=gamma,
DocTopicCount=DocTopicCount,
theta=theta,
thetaRem=thetaRem,
sumLogPi=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
rho=rho,
omega=omega)
if sumLogPiActiveVec is None:
sumLogPiActiveVec, sumLogPiRemVec, LP = DocTopicCount_to_sumLogPi(
rho=rho,
omega=omega,
DocTopicCount=DocTopicCount,
alpha=alpha,
gamma=gamma)
Lrhoomega = OptimizerRhoOmegaBetter.negL_rhoomega_viaHDPTopicUtil(
nDoc=nDoc,
alpha=alpha,
gamma=gamma,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
rho=rho,
omega=omega)
if f is None:
f = Lrhoomega
assert np.allclose(f, Lrhoomega)
print("%10s Ltro= % .8e Lro= % .5e fro= % .5e" % (msg, L, Lrhoomega, f))
return L
Tvec = np.asarray(
[-30324.1449, -99350.3919, -135508.8616, -135988.3834, -143709.5131])
K = Tvec.size
Uvec = np.zeros(K)
'''
betaSORT = np.asarray([0.3, 0.2, 0.1])
omega = OROB.make_initomega(K, nDoc, gamma)
print L_hdp(betaSORT, omega, Tvec, alpha)
betaUNSORT = np.asarray([0.2, 0.3, 0.1])
print L_hdp(betaUNSORT, omega, Tvec, alpha)
'''
# Find optimal rho
rho, omega, f, Info = OROB.find_optimum(alpha=alpha,
gamma=gamma,
nDoc=nDoc,
sumLogPiActiveVec=Tvec,
sumLogPiRemVec=Uvec)
beta_opt = OROB.rho2beta_active(rho)
Lbest = L_hdp(beta_opt, omega, Tvec, alpha)
assert np.allclose(-1 * f, Lbest)
sortIDs = argsort_bigtosmall_stable(Tvec)
Lbest_sorted = L_hdp(beta_opt[sortIDs], omega, Tvec[sortIDs], alpha)
print(" % .5e Lbest" % (Lbest))
print(" % .5e Lbest after sorting" % (Lbest_sorted))
print(beta_opt)
print(beta_opt[sortIDs])
def learn_rhoomega_fromFixedCounts(DocTopicCount=None,
nDoc=0,
canShuffleInit='byUsage',
canShuffle=None,
maxiter=5,
warmStart_rho=1,
alpha=None, gamma=None,
initrho=None, initomega=None, **kwargs):
assert nDoc == DocTopicCount.shape[0]
K = DocTopicCount.shape[1]
didShuffle = 0
if canShuffleInit:
if canShuffleInit.lower().count('byusage'):
print 'INITIAL SORTING BY USAGE'
avgPi = calcAvgPiFromDocTopicCount(DocTopicCount)
bigtosmall = argsort_bigtosmall_stable(avgPi)
elif canShuffleInit.lower().count('bycount'):
print 'INITIAL SORTING BY COUNT'
bigtosmall = argsort_bigtosmall_stable(DocTopicCount.sum(axis=0))
elif canShuffleInit.lower().count('random'):
print 'INITIAL SORTING RANDOMLY'
PRNG = np.random.RandomState(0)
bigtosmall = np.arange(K)
PRNG.shuffle(bigtosmall)
else:
bigtosmall = np.arange(K)
# Now, sort.
if not np.allclose(bigtosmall, np.arange(K)):
DocTopicCount = DocTopicCount[:, bigtosmall]
didShuffle = 1
avgPi = calcAvgPiFromDocTopicCount(DocTopicCount)
sortedids = argsort_bigtosmall_stable(avgPi)
if canShuffleInit.lower().count('byusage'):
assert np.allclose(sortedids, np.arange(K))
# Find UIDs of comps to track
emptyUIDs = np.flatnonzero(DocTopicCount.sum(axis=0) < 0.0001)
if emptyUIDs.size >= 3:
firstEmptyUID = emptyUIDs.min()
lastEmptyUID = emptyUIDs.max()
middleEmptyUID = emptyUIDs[len(emptyUIDs)/2]
trackEmptyUIDs = [firstEmptyUID, middleEmptyUID, lastEmptyUID]
emptyLabels = ['first', 'middle', 'last']
elif emptyUIDs.size == 2:
trackEmptyUIDs = [emptyUIDs.min(), emptyUIDs.max()]
emptyLabels = ['first', 'last']
elif emptyUIDs.size == 1:
firstEmptyUID = emptyUIDs.min()
trackEmptyUIDs = [firstEmptyUID]
emptyLabels = ['first']
else:
trackEmptyUIDs = []
emptyLabels = []
trackActiveUIDs = list()
activeLabels = list()
# Track the top 5 active columns of DocTopicCount
for pos in range(0, np.minimum(5, K)):
if sortedids[pos] not in emptyUIDs:
trackActiveUIDs.append(sortedids[pos])
activeLabels.append('max+%d' % (pos))
# Find the minnonemptyID
for pos in range(K-1, 0, -1):
curid = sortedids[pos]
if curid not in emptyUIDs:
break
minnonemptyPos = pos
# Track the 5 smallest active columns of DocTopicCount
nBeyond5 = np.minimum(5, K - len(emptyUIDs) - 5)
for i in range(-1 * (nBeyond5-1), 1):
trackActiveUIDs.append(sortedids[minnonemptyPos + i])
activeLabels.append('min+%d' % (-1 * i))
assert np.all(avgPi[trackActiveUIDs] > 0)
assert np.allclose(avgPi[trackEmptyUIDs], 0.0)
assert is_sorted_bigtosmall(avgPi[trackActiveUIDs])
nDocToDisplay = np.minimum(nDoc, 10)
# Initialize rho
if initrho is None:
rho = OptimizerRhoOmegaBetter.make_initrho(K, nDoc, gamma)
else:
if didShuffle:
rho, _ = reorder_rho(initrho, bigtosmall)
else:
rho = initrho
# Initialize omega
if initomega is None:
omega = OptimizerRhoOmegaBetter.make_initomega(K, nDoc, gamma)
else:
omega = initomega
# ELBO value of initial state
Ltro = evalELBOandPrint(
rho=rho, omega=omega,
nDoc=nDoc,
DocTopicCount=DocTopicCount,
alpha=alpha, gamma=gamma,
msg='init',
)
Snapshots = dict()
Snapshots['DTCSum'] = list()
Snapshots['DTCUsage'] = list()
Snapshots['beta'] = list()
Snapshots['Lscore'] = list()
Snapshots['activeLabels'] = activeLabels
Snapshots['emptyLabels'] = emptyLabels
Snapshots['pos_trackActive'] = list()
Snapshots['pos_trackEmpty'] = list()
Snapshots['beta_trackActive'] = list()
Snapshots['beta_trackEmpty'] = list()
Snapshots['count_trackActive'] = list()
Snapshots['count_trackEmpty'] = list()
Snapshots['beta_trackRem'] = list()
LtroList = list()
LtroList.append(Ltro)
betaK = rho2beta(rho, returnSize="K")
iterid = 0
prevbetaK = np.zeros_like(betaK)
prevrho = rho.copy()
while np.sum(np.abs(betaK - prevbetaK)) > 0.0000001:
iterid += 1
if iterid > maxiter:
break
# Take Snapshots of Learned Params
Snapshots['Lscore'].append(Ltro)
Snapshots['DTCSum'].append(DocTopicCount.sum(axis=0))
Snapshots['DTCUsage'].append((DocTopicCount > 0.001).sum(axis=0))
Snapshots['beta'].append(betaK)
Snapshots['pos_trackActive'].append(trackActiveUIDs)
Snapshots['pos_trackEmpty'].append(trackEmptyUIDs)
Snapshots['beta_trackActive'].append(betaK[trackActiveUIDs])
Snapshots['beta_trackEmpty'].append(betaK[trackEmptyUIDs])
Snapshots['beta_trackRem'].append(1.0 - betaK.sum())
Snapshots['count_trackActive'].append(
DocTopicCount.sum(axis=0)[trackActiveUIDs])
Snapshots['count_trackEmpty'].append(
DocTopicCount.sum(axis=0)[trackEmptyUIDs])
# Sort by beta
didShuffle = 0
tlabel = '_t'
if iterid > 1 and canShuffle and canShuffle.lower().count('bybeta'):
bigtosmall = argsort_bigtosmall_stable(betaK)
if not np.allclose(bigtosmall, np.arange(K)):
trackActiveUIDs = mapToNewPos(trackActiveUIDs, bigtosmall)
trackEmptyUIDs = mapToNewPos(trackEmptyUIDs, bigtosmall)
rho, betaK = reorder_rho(rho, bigtosmall)
DocTopicCount = DocTopicCount[:, bigtosmall]
didShuffle = 1
tlabel = '_ts'
# Update theta
sumLogPiActiveVec, sumLogPiRemVec, LP = DocTopicCount_to_sumLogPi(
rho=rho, omega=omega,
DocTopicCount=DocTopicCount,
alpha=alpha, gamma=gamma,
**kwargs)
# Show ELBO with freshly-optimized theta value.
Ltro = evalELBOandPrint(
rho=rho, omega=omega,
DocTopicCount=DocTopicCount,
theta=LP['theta'],
thetaRem=LP['thetaRem'],
nDoc=nDoc,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
alpha=alpha, gamma=gamma, f=None,
msg=str(iterid) + tlabel,
)
LtroList.append(Ltro)
if not LtroList[-1] >= LtroList[-2]:
if didShuffle:
print 'NOT MONOTONIC! just after theta update with SHUFFLE!'
else:
print 'NOT MONOTONIC! just after theta standard update'
didELBODrop = 0
if canShuffle:
if canShuffle.lower().count('bysumlogpi'):
bigtosmall = argsort_bigtosmall_stable(
sumLogPiActiveVec)
elif canShuffle.lower().count('bycounts'):
bigtosmall = argsort_bigtosmall_stable(
DocTopicCount.sum(axis=0))
elif canShuffle.lower().count('byusage'):
estPi = DocTopicCount / DocTopicCount.sum(axis=1)[:,np.newaxis]
avgPi = np.sum(estPi, axis=0)
bigtosmall = argsort_bigtosmall_stable(avgPi)
else:
bigtosmall = np.arange(K)
if not np.allclose(bigtosmall, np.arange(K)):
trackActiveUIDs = mapToNewPos(trackActiveUIDs, bigtosmall)
trackEmptyUIDs = mapToNewPos(trackEmptyUIDs, bigtosmall)
rho, betaK = reorder_rho(rho, bigtosmall)
sumLogPiActiveVec = sumLogPiActiveVec[bigtosmall]
DocTopicCount = DocTopicCount[:,bigtosmall]
LP['theta'] = LP['theta'][:, bigtosmall]
didShuffle = 1
# Show ELBO with freshly-optimized rho value.
Ltro = evalELBOandPrint(
rho=rho, omega=omega,
DocTopicCount=DocTopicCount,
theta=LP['theta'],
thetaRem=LP['thetaRem'],
nDoc=nDoc,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
alpha=alpha, gamma=gamma, f=None,
msg=str(iterid) + "_ss",
)
LtroList.append(Ltro)
if not LtroList[-1] >= LtroList[-2]:
print 'NOT MONOTONIC! just after %s shuffle update!' % (
canShuffle)
didELBODrop = 1
prevrho[:] = rho
# Update rhoomega
if warmStart_rho:
initrho = rho
else:
initrho = None
rho, omega, f, Info = OptimizerRhoOmegaBetter.\
find_optimum_multiple_tries(
alpha=alpha,
gamma=gamma,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
nDoc=nDoc,
initrho=initrho,
initomega=omega,
approx_grad=1,
do_grad_omega=0,
)
prevbetaK[:] = betaK
betaK = rho2beta(rho, returnSize="K")
# Show ELBO with freshly-optimized rho value.
Ltro = evalELBOandPrint(
rho=rho, omega=omega,
DocTopicCount=DocTopicCount,
theta=LP['theta'],
thetaRem=LP['thetaRem'],
nDoc=nDoc,
sumLogPiActiveVec=sumLogPiActiveVec,
sumLogPiRemVec=sumLogPiRemVec,
alpha=alpha, gamma=gamma, f=f,
msg=str(iterid) + "_r",
)
LtroList.append(Ltro)
if not LtroList[-1] >= LtroList[-2]:
print 'NOT MONOTONIC! just after rho update!'
if didELBODrop:
if LtroList[-1] >= LtroList[-3]:
print 'Phew. Combined update of sorting then optimizing rho OK'
else:
print 'WHOA! Combined update of sorting then' + \
' optimizing rho beta NOT MONOTONIC'
Snapshots['Lscore'].append(Ltro)
Snapshots['DTCSum'].append(DocTopicCount.sum(axis=0))
Snapshots['DTCUsage'].append((DocTopicCount > 0.001).sum(axis=0))
Snapshots['beta'].append(betaK)
Snapshots['pos_trackActive'].append(trackActiveUIDs)
Snapshots['pos_trackEmpty'].append(trackEmptyUIDs)
Snapshots['beta_trackActive'].append(betaK[trackActiveUIDs])
Snapshots['beta_trackEmpty'].append(betaK[trackEmptyUIDs])
Snapshots['beta_trackRem'].append(1.0 - betaK.sum())
Snapshots['count_trackActive'].append(
DocTopicCount.sum(axis=0)[trackActiveUIDs])
Snapshots['count_trackEmpty'].append(
DocTopicCount.sum(axis=0)[trackEmptyUIDs])
print '\nEmpty cluster ids (%d of %d)' % (
len(trackEmptyUIDs), len(emptyUIDs))
print '-----------------'
print ' '.join(['% 10d' % (x) for x in trackEmptyUIDs])
print '\nSelected active clusters to track'
print '---------------------------------'
print ' '.join(['% 10d' % (x) for x in trackActiveUIDs])
print ' '.join(['% .3e' % (x) for x in avgPi[trackActiveUIDs]])
print '\nDocTopicCount for %d of %d docs' % (nDocToDisplay, nDoc)
print '---------------------------------'
for n in range(nDocToDisplay):
print ' '.join([
'% 9.2f' % (x) for x in DocTopicCount[n, trackActiveUIDs]])
print '\nFinal sumLogPiActiveVec'
print '---------------------------------'
print ' '.join(['% .3e' % (x) for x in sumLogPiActiveVec[trackActiveUIDs]])
print 'is sumLogPiActiveVec sorted?', \
is_sorted_bigtosmall(sumLogPiActiveVec)
return rho, omega, Snapshots
def reorder_rho(rho, bigtosmallIDs):
betaK = rho2beta(rho, returnSize='K')
newbetaK = betaK[bigtosmallIDs]
return OptimizerRhoOmegaBetter.beta2rho(newbetaK, rho.size), newbetaK